"""Helper operations and classes for general model building.
"""
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals

import collections
import pickle
import os
import time
import warnings
import numpy as np
import pandas as pd
import tensorflow as tf
import tempfile
from deepchem.models import Model
from deepchem.metrics import from_one_hot
from deepchem.nn import model_ops
from deepchem.models.tensorflow_models import utils as tf_utils
from deepchem.utils.save import log
from deepchem.data import pad_features
from tensorflow.contrib.layers.python.layers import batch_norm

def softmax(x):
  """Simple numpy softmax implementation
  """
  # (n_samples, n_classes)
  if len(x.shape) == 2:
    row_max = np.max(x, axis = 1)
    x -= row_max.reshape((x.shape[0], 1))
    x = np.exp(x)
    row_sum = np.sum(x, axis = 1)
    x /= row_sum.reshape((x.shape[0], 1))
  # (n_samples, n_tasks, n_classes)
  elif len(x.shape) == 3:
    row_max = np.max(x, axis = 2)
    x -= row_max.reshape(x.shape[:2] + (1,))
    x = np.exp(x)
    row_sum = np.sum(x, axis = 2)
    x /= row_sum.reshape(x.shape[:2] + (1,))
  return x

class TensorflowGraph(object):
  """Simple class that holds information needed to run Tensorflow graph."""
  def __init__(self, graph, session, name_scopes, output, labels, weights, loss):
    self.graph = graph
    self.session = session
    self.name_scopes = name_scopes
    self.output = output
    self.labels = labels
    self.weights = weights
    self.loss = loss

  @staticmethod
  def get_placeholder_scope(graph, name_scopes):
    """Gets placeholder scope."""
    placeholder_root = "placeholders"
    return TensorflowGraph.shared_name_scope(placeholder_root, graph, name_scopes)

  @staticmethod
  def shared_name_scope(name, graph, name_scopes):
    """Returns a singleton TensorFlow scope with the given name.

    Used to prevent '_1'-appended scopes when sharing scopes with child classes.

    Args:
      name: String. Name scope for group of operations.
    Returns:
      tf.name_scope with the provided name.
    """
    with graph.as_default():
      if name not in name_scopes:
        with tf.name_scope(name) as scope:
          name_scopes[name] = scope
      return tf.name_scope(name_scopes[name])

  @staticmethod
  def get_feed_dict(named_values):
    feed_dict = {}
    placeholder_root = "placeholders"
    for name, value in named_values.items():
      feed_dict['{}/{}:0'.format(placeholder_root, name)] = value
    return feed_dict

class TensorflowGraphModel(Model):
  """Parent class for deepchem Tensorflow models.

  Classifier:
    n_classes

  Has the following attributes:

    placeholder_root: String placeholder prefix, used to create
      placeholder_scope.

  Generic base class for defining, training, and evaluating TensorflowGraphs.

  Subclasses must implement the following methods:
    build
    add_output_ops
    add_training_cost

  Args:
    train: If True, model is in training mode.
    logdir: Directory for output files.
  """

  def __init__(self, n_tasks, n_features, logdir=None, layer_sizes=[1000],
               weight_init_stddevs=[.02], bias_init_consts=[1.], penalty=0.0,
               penalty_type="l2", dropouts=[0.5], learning_rate=.001,
               momentum=.9, optimizer="adam", batch_size=50, n_classes=2,
               verbose=True, seed=None, **kwargs):
    """Constructs the computational graph.

    This function constructs the computational graph for the model. It relies
    subclassed methods (build/cost) to construct specific graphs.

    Parameters
    ----------
    n_tasks: int
      Number of tasks
    n_features: int
      Number of features.
    logdir: str
      Location to save data
    layer_sizes: list
      List of layer sizes.
    weight_init_stddevs: list
      List of standard deviations for weights (sampled from zero-mean
      gaussians). One for each layer.
    bias_init_consts: list
      List of bias initializations. One for each layer.
    penalty: float
      Amount of penalty (l2 or l1 applied)
    penalty_type: str
      Either "l2" or "l1"
    dropouts: list
      List of dropout amounts. One for each layer.
    learning_rate: float
      Learning rate for model.
    momentum: float
      Momentum. Only applied if optimizer=="momentum"
    optimizer: str
      Type of optimizer applied.
    batch_size: int
      Size of minibatches for training.
    n_classes: int
      Number of classes if this is for classification.
      TODO(rbharath): Move this argument to TensorflowClassifier
    verbose: True
      Perform logging.
    seed: int
      If not none, is used as random seed for tensorflow.
    """
    # Save hyperparameters
    self.n_tasks = n_tasks
    self.n_features = n_features
    self.layer_sizes = layer_sizes
    self.weight_init_stddevs = weight_init_stddevs
    self.bias_init_consts = bias_init_consts
    self.penalty = penalty
    self.penalty_type = penalty_type
    self.dropouts = dropouts
    self.learning_rate = learning_rate
    self.momentum = momentum
    self.optimizer = optimizer
    self.batch_size = batch_size
    self.n_classes = n_classes
    self.verbose= verbose
    self.seed = seed

    if logdir is not None:
      if not os.path.exists(logdir):
        os.makedirs(logdir)
    else:
      logdir = tempfile.mkdtemp()
    self.logdir = logdir

    # Guard variable to make sure we don't Restore() this model
    # from a disk checkpoint more than once.
    self._restored_model = False
    # Path to save checkpoint files, which matches the
    # replicated supervisor's default path.
    self._save_path = os.path.join(logdir, 'model.ckpt')

    self.train_graph = self.construct_graph(training=True, seed=self.seed)
    self.eval_graph = self.construct_graph(training=False, seed=self.seed)

  def save(self):
    """
    No-op since tf models save themselves during fit()
    """
    pass

  def reload(self):
    """
    Loads model from disk. Thin wrapper around restore() for consistency.
    """
    self.restore()

  def get_num_tasks(self):
    return self.n_tasks

  def construct_graph(self, training, seed):
    """Returns a TensorflowGraph object."""
    graph = tf.Graph()

    # Lazily created by _get_shared_session().
    shared_session = None

    # Cache of TensorFlow scopes, to prevent '_1' appended scope names
    # when subclass-overridden methods use the same scopes.
    name_scopes = {}

    # Setup graph
    with graph.as_default():
      if seed is not None:
        tf.set_random_seed(seed)
      output = self.build(graph, name_scopes, training)
      labels = self.add_label_placeholders(graph, name_scopes)
      weights = self.add_example_weight_placeholders(graph, name_scopes)

    if training:
      loss = self.add_training_cost(graph, name_scopes, output, labels, weights)
    else:
      loss = None
      output = self.add_output_ops(graph, output)  # add softmax heads
    return TensorflowGraph(graph=graph,
                           session=shared_session,
                           name_scopes=name_scopes,
                           output=output,
                           labels=labels,
                           weights=weights,
                           loss=loss)

  def add_training_cost(self, graph, name_scopes, output, labels, weights):
    with graph.as_default():
      epsilon = 1e-3  # small float to avoid dividing by zero
      weighted_costs = []  # weighted costs for each example
      gradient_costs = []  # costs used for gradient calculation

      with TensorflowGraph.shared_name_scope('costs', graph, name_scopes):
        for task in range(self.n_tasks):
          task_str = str(task).zfill(len(str(self.n_tasks)))
          with TensorflowGraph.shared_name_scope(
              'cost_{}'.format(task_str), graph, name_scopes):
            with tf.name_scope('weighted'):
              weighted_cost = self.cost(output[task], labels[task],
                                        weights[task])
              weighted_costs.append(weighted_cost)

            with tf.name_scope('gradient'):
              # Note that we divide by the batch size and not the number of
              # non-zero weight examples in the batch.  Also, instead of using
              # tf.reduce_mean (which can put ops on the CPU) we explicitly
              # calculate with div/sum so it stays on the GPU.
              gradient_cost = tf.div(tf.reduce_sum(weighted_cost),
                                     self.batch_size)
              gradient_costs.append(gradient_cost)

        # aggregated costs
        with TensorflowGraph.shared_name_scope('aggregated', graph, name_scopes):
          with tf.name_scope('gradient'):
            loss = tf.add_n(gradient_costs)

          # weight decay
          if self.penalty != 0.0:
            penalty = model_ops.weight_decay(self.penalty_type, self.penalty)
            loss += penalty

      return loss

  def fit(self, dataset, nb_epoch=10, pad_batches=False,
          max_checkpoints_to_keep=5, log_every_N_batches=50, **kwargs):
    """Fit the model.

    Parameters
    ----------
    dataset: dc.data.Dataset
      Dataset object holding training data
    nb_epoch: 10
      Number of training epochs.
    pad_batches: bool
      Whether or not to pad each batch to exactly be of size batch_size.
    max_checkpoints_to_keep: int
      Maximum number of checkpoints to keep; older checkpoints will be deleted.
    log_every_N_batches: int
      Report every N batches. Useful for training on very large datasets,
      where epochs can take long time to finish.

    Raises
    ------
    AssertionError
      If model is not in training mode.
    """
    ############################################################## TIMING
    time1 = time.time()
    ############################################################## TIMING
    log("Training for %d epochs" % nb_epoch, self.verbose)
    with self.train_graph.graph.as_default():
      train_op = self.get_training_op(
          self.train_graph.graph, self.train_graph.loss)
      with self._get_shared_session(train=True) as sess:
        sess.run(tf.global_variables_initializer())
        saver = tf.train.Saver(max_to_keep=max_checkpoints_to_keep)
        # Save an initial checkpoint.
        saver.save(sess, self._save_path, global_step=0)
        for epoch in range(nb_epoch):
          avg_loss, n_batches = 0., 0
          for ind, (X_b, y_b, w_b, ids_b) in enumerate(
              # Turns out there are valid cases where we don't want pad-batches
              # on by default.
              #dataset.iterbatches(batch_size, pad_batches=True)):
              dataset.iterbatches(self.batch_size, pad_batches=pad_batches)):
            if ind % log_every_N_batches == 0:
              log("On batch %d" % ind, self.verbose)
            # Run training op.
            feed_dict = self.construct_feed_dict(X_b, y_b, w_b, ids_b)
            fetches = self.train_graph.output + [
                train_op, self.train_graph.loss]
            fetched_values = sess.run(fetches, feed_dict=feed_dict)
            output = fetched_values[:len(self.train_graph.output)]
            loss = fetched_values[-1]
            avg_loss += loss
            y_pred = np.squeeze(np.array(output))
            y_b = y_b.flatten()
            n_batches += 1
          saver.save(sess, self._save_path, global_step=epoch)
          avg_loss = float(avg_loss)/n_batches
          log('Ending epoch %d: Average loss %g' % (epoch, avg_loss), self.verbose)
        # Always save a final checkpoint when complete.
        saver.save(sess, self._save_path, global_step=epoch+1)
    ############################################################## TIMING
    time2 = time.time()
    print("TIMING: model fitting took %0.3f s" % (time2-time1),
          self.verbose)
    ############################################################## TIMING

  def add_output_ops(self, graph, output):
    """Replace logits with softmax outputs."""
    with graph.as_default():
      softmax = []
      with tf.name_scope('inference'):
        for i, logits in enumerate(output):
          softmax.append(tf.nn.softmax(logits, name='softmax_%d' % i))
      output = softmax
    return output

  def build(self, graph, name_scopes, training):
    """Define the core graph.

    NOTE(user): Operations defined here should be in their own name scope to
    avoid any ambiguity when restoring checkpoints.
    Raises:
      NotImplementedError: if not overridden by concrete subclass.
    """
    raise NotImplementedError('Must be overridden by concrete subclass')

  def construct_feed_dict(self, X_b, y_b=None, w_b=None, ids_b=None):
    """Transform a minibatch of data into a feed_dict.

    Raises:
      NotImplementedError: if not overridden by concrete subclass.
    """
    raise NotImplementedError('Must be overridden by concrete subclass')

  def add_label_placeholders(self, graph, name_scopes):
    """Add Placeholders for labels for each task.

    This method creates the following Placeholders for each task:
      labels_%d: Float label tensor. For classification tasks, this tensor will
        have shape batch_size x n_classes. For regression tasks, this tensor
        will have shape batch_size.

    Raises:
      NotImplementedError: if not overridden by concrete subclass.
    """
    raise NotImplementedError('Must be overridden by concrete subclass')

  def add_example_weight_placeholders(self, graph, name_scopes):
    """Add Placeholders for example weights for each task.

    This method creates the following Placeholders for each task:
      weights_%d: Label tensor with shape batch_size.

    Placeholders are wrapped in identity ops to avoid the error caused by
    feeding and fetching the same tensor.
    """
    weights = []
    placeholder_scope = TensorflowGraph.get_placeholder_scope(graph, name_scopes)
    with placeholder_scope:
      for task in range(self.n_tasks):
        weights.append(tf.identity(
            tf.placeholder(tf.float32, shape=[None],
                           name='weights_%d' % task)))
    return weights

  def cost(self, output, labels, weights):
    """Calculate single-task training cost for a batch of examples.

    Args:
      output: Tensor with model outputs.
      labels: Tensor with true labels.
      weights: Tensor with shape batch_size containing example weights.

    Returns:
      A tensor with shape batch_size containing the weighted cost for each
      example. For use in subclasses that want to calculate additional costs.
    """
    raise NotImplementedError('Must be overridden by concrete subclass')

  def get_training_op(self, graph, loss):
    """Get training op for applying gradients to variables.

    Subclasses that need to do anything fancy with gradients should override
    this method.

    Returns:
    A training op.
    """
    with graph.as_default():
      opt = model_ops.optimizer(self.optimizer, self.learning_rate, self.momentum)
      return opt.minimize(loss, name='train')

  def _get_shared_session(self, train):
    # allow_soft_placement=True allows ops without a GPU implementation
    # to run on the CPU instead.
    if train:
      if not self.train_graph.session:
        config = tf.ConfigProto(allow_soft_placement=True)
        self.train_graph.session = tf.Session(config=config)
      return self.train_graph.session
    else:
      if not self.eval_graph.session:
        config = tf.ConfigProto(allow_soft_placement=True)
        self.eval_graph.session = tf.Session(config=config)
      return self.eval_graph.session

  def restore(self):
    """Restores the model from the provided training checkpoint.

    Args:
      checkpoint: string. Path to checkpoint file.
    """
    if self._restored_model:
      return
    with self.eval_graph.graph.as_default():
      last_checkpoint = self._find_last_checkpoint()
      # TODO(rbharath): Is setting train=False right here?
      saver = tf.train.Saver()
      saver.restore(self._get_shared_session(train=False),
                    last_checkpoint)
      self._restored_model = True

  def _find_last_checkpoint(self):
    """Finds last saved checkpoint."""
    highest_num, last_checkpoint = -np.inf, None
    for filename in os.listdir(self.logdir):
      # checkpoints look like logdir/model.ckpt-N
      # self._save_path is "logdir/model.ckpt"
      if os.path.basename(self._save_path) in filename:
        try:
          N = int(filename.split("-")[1].split(".")[0])
          if N > highest_num:
            highest_num = N
            last_checkpoint = "model.ckpt-"+str(N)
        except ValueError:
          pass
    return os.path.join(self.logdir, last_checkpoint)

class TensorflowClassifier(TensorflowGraphModel):
  """Classification model.

  Subclasses must set the following attributes:
    output: logits op(s) used for computing classification loss and predicted
      class probabilities for each task.
  """
  def get_task_type(self):
    return "classification"

  def cost(self, logits, labels, weights):
    """Calculate single-task training cost for a batch of examples.

    Args:
      logits: Tensor with shape batch_size x n_classes containing logits.
      labels: Tensor with shape batch_size x n_classes containing true labels
        in a one-hot encoding.
      weights: Tensor with shape batch_size containing example weights.

    Returns:
      A tensor with shape batch_size containing the weighted cost for each
      example.
    """
    return tf.mul(tf.nn.softmax_cross_entropy_with_logits(logits, labels),
                  weights)

  def add_label_placeholders(self, graph, name_scopes):
    """Add Placeholders for labels for each task.

    This method creates the following Placeholders for each task:
      labels_%d: Label tensor with shape batch_size x n_classes.

    Placeholders are wrapped in identity ops to avoid the error caused by
    feeding and fetching the same tensor.
    """
    placeholder_scope = TensorflowGraph.get_placeholder_scope(graph, name_scopes)
    with graph.as_default():
      batch_size = self.batch_size
      n_classes = self.n_classes
      labels = []
      with placeholder_scope:
        for task in range(self.n_tasks):
          labels.append(tf.identity(
              tf.placeholder(tf.float32, shape=[None, n_classes],
                             name='labels_%d' % task)))
      return labels

  def predict_on_batch(self, X, pad_batch=False):
    """Return model output for the provided input.

    Restore(checkpoint) must have previously been called on this object.

    Args:
      dataset: dc.data.dataset object.

    Returns:
      Tuple of three numpy arrays with shape n_examples x n_tasks (x ...):
        output: Model outputs.
        labels: True labels.
        weights: Example weights.
      Note that the output and labels arrays may be more than 2D, e.g. for
      classifier models that return class probabilities.

    Raises:
      AssertionError: If model is not in evaluation mode.
      ValueError: If output and labels are not both 3D or both 2D.
    """
    len_unpadded = len(X)
    if pad_batch:
      X = pad_features(self.batch_size, X)

    if not self._restored_model:
      self.restore()
    with self.eval_graph.graph.as_default():

      # run eval data through the model
      n_tasks = self.n_tasks
      output = []
      with self._get_shared_session(train=False).as_default():
        feed_dict = self.construct_feed_dict(X)
        data = self._get_shared_session(train=False).run(
            self.eval_graph.output, feed_dict=feed_dict)
        batch_output = np.asarray(data[:n_tasks], dtype=float)
        # reshape to batch_size x n_tasks x ...
        if batch_output.ndim == 3:
          batch_output = batch_output.transpose((1, 0, 2))
        elif batch_output.ndim == 2:
          batch_output = batch_output.transpose((1, 0))
        else:
          raise ValueError(
              'Unrecognized rank combination for output: %s' %
              (batch_output.shape,))
        output.append(batch_output)

        outputs = np.array(from_one_hot(
            np.squeeze(np.concatenate(output)), axis=-1))

    outputs = np.copy(outputs)
    outputs = np.reshape(outputs, (len(X), n_tasks))


# Copyright 1999-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2

import gzip
import io
import sys
import tempfile

from _emerge.AsynchronousLock import AsynchronousLock
from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor
from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
from _emerge.EbuildProcess import EbuildProcess
from _emerge.CompositeTask import CompositeTask
from portage.package.ebuild.prepare_build_dirs import _prepare_workdir
from portage.util import writemsg

try:
	from portage.xml.metadata import MetaDataXML
except (SystemExit, KeyboardInterrupt):
	raise
except (ImportError, SystemError, RuntimeError, Exception):
	# broken or missing xml support
	# http://bugs.python.org/issue14988
	MetaDataXML = None

import portage
portage.proxy.lazyimport.lazyimport(globals(),
	'portage.elog:messages@elog_messages',
	'portage.package.ebuild.doebuild:_check_build_log,' + \
		'_post_phase_cmds,_post_phase_userpriv_perms,' + \
		'_post_src_install_soname_symlinks,' + \
		'_post_src_install_uid_fix,_postinst_bsdflags,' + \
		'_post_src_install_write_metadata,' + \
		'_preinst_bsdflags'
)
from portage import os
from portage import _encodings
from portage import _unicode_encode

class EbuildPhase(CompositeTask):

	__slots__ = ("actionmap", "phase", "settings") + \
		("_ebuild_lock",)

	# FEATURES displayed prior to setup phase
	_features_display = (
		"ccache", "compressdebug", "distcc", "distcc-pump", "fakeroot",
		"installsources", "keeptemp", "keepwork", "nostrip",
		"preserve-libs", "sandbox", "selinux", "sesandbox",
		"splitdebug", "suidctl", "test", "userpriv",
		"usersandbox"
	)

	# Locked phases
	_locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm")

	def _start(self):

		need_builddir = self.phase not in EbuildProcess._phases_without_builddir

		if need_builddir:
			phase_completed_file = os.path.join(
				self.settings['PORTAGE_BUILDDIR'],
				".%sed" % self.phase.rstrip('e'))
			if not os.path.exists(phase_completed_file):
				# If the phase is really going to run then we want
				# to eliminate any stale elog messages that may
				# exist from a previous run.
				try:
					os.unlink(os.path.join(self.settings['T'],
						'logging', self.phase))
				except OSError:
					pass

		if self.phase in ('nofetch', 'pretend', 'setup'):

			use = self.settings.get('PORTAGE_BUILT_USE')
			if use is None:
				use = self.settings['PORTAGE_USE']

			maint_str = ""
			upstr_str = ""
			metadata_xml_path = os.path.join(os.path.dirname(self.settings['EBUILD']), "metadata.xml")
			if MetaDataXML is not None and os.path.isfile(metadata_xml_path):
				herds_path = os.path.join(self.settings['PORTDIR'],
					'metadata/herds.xml')
				try:
					metadata_xml = MetaDataXML(metadata_xml_path, herds_path)
					maint_str = metadata_xml.format_maintainer_string()
					upstr_str = metadata_xml.format_upstream_string()
				except SyntaxError:
					maint_str = "<invalid metadata.xml>"

			msg = []
			msg.append("Package:    %s" % self.settings.mycpv)
			if self.settings.get('PORTAGE_REPO_NAME'):
				msg.append("Repository: %s" % self.settings['PORTAGE_REPO_NAME'])
			if maint_str:
				msg.append("Maintainer: %s" % maint_str)
			if upstr_str:
				msg.append("Upstream:   %s" % upstr_str)

			msg.append("USE:        %s" % use)
			relevant_features = []
			enabled_features = self.settings.features
			for x in self._features_display:
				if x in enabled_features:
					relevant_features.append(x)
			if relevant_features:
				msg.append("FEATURES:   %s" % " ".join(relevant_features))

			# Force background=True for this header since it's intended
			# for the log and it doesn't necessarily need to be visible
			# elsewhere.
			self._elog('einfo', msg, background=True)

		if self.phase == 'package':
			if 'PORTAGE_BINPKG_TMPFILE' not in self.settings:
				self.settings['PORTAGE_BINPKG_TMPFILE'] = \
					os.path.join(self.settings['PKGDIR'],
					self.settings['CATEGORY'], self.settings['PF']) + '.tbz2'

		if self.phase in ("pretend", "prerm"):
			env_extractor = BinpkgEnvExtractor(background=self.background,
				scheduler=self.scheduler, settings=self.settings)
			if env_extractor.saved_env_exists():
				self._start_task(env_extractor, self._env_extractor_exit)
				return
			# If the environment.bz2 doesn't exist, then ebuild.sh will
			# source the ebuild as a fallback.

		self._start_lock()

	def _env_extractor_exit(self, env_extractor):
		if self._default_exit(env_extractor) != os.EX_OK:
			self.wait()
			return

		self._start_lock()

	def _start_lock(self):
		if (self.phase in self._locked_phases and
			"ebuild-locks" in self.settings.features):
			eroot = self.settings["EROOT"]
			lock_path = os.path.join(eroot, portage.VDB_PATH + "-ebuild")
			if os.access(os.path.dirname(lock_path), os.W_OK):
				self._ebuild_lock = AsynchronousLock(path=lock_path,
					scheduler=self.scheduler)
				self._start_task(self._ebuild_lock, self._lock_exit)
				return

		self._start_ebuild()

	def _lock_exit(self, ebuild_lock):
		if self._default_exit(ebuild_lock) != os.EX_OK:
			self.wait()
			return
		self._start_ebuild()

	def _start_ebuild(self):

		# Don't open the log file during the clean phase since the
		# open file can result in an nfs lock on $T/build.log which
		# prevents the clean phase from removing $T.
		logfile = None
		if self.phase not in ("clean", "cleanrm") and \
			self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
			logfile = self.settings.get("PORTAGE_LOG_FILE")

		fd_pipes = None
		if not self.background and self.phase == 'nofetch':
			# All the pkg_nofetch output goes to stderr since
			# it's considered to be an error message.
			fd_pipes = {1 : sys.stderr.fileno()}

		ebuild_process = EbuildProcess(actionmap=self.actionmap,
			background=self.background, fd_pipes=fd_pipes, logfile=logfile,
			phase=self.phase, scheduler=self.scheduler,
			settings=self.settings)

		self._start_task(ebuild_process, self._ebuild_exit)

	def _ebuild_exit(self, ebuild_process):

		if self._ebuild_lock is not None:
			self._ebuild_lock.unlock()
			self._ebuild_lock = None

		fail = False
		if self._default_exit(ebuild_process) != os.EX_OK:
			if self.phase == "test" and \
				"test-fail-continue" in self.settings.features:
				pass
			else:
				fail = True

		if not fail:
			self.returncode = None

		logfile = None
		if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
			logfile = self.settings.get("PORTAGE_LOG_FILE")

		if self.phase == "install":
			out = io.StringIO()
			_check_build_log(self.settings, out=out)
			msg = out.getvalue()
			self.scheduler.output(msg, log_path=logfile)

		if fail:
			self._die_hooks()
			return

		settings = self.settings
		_post_phase_userpriv_perms(settings)

		if self.phase == "unpack":
			# Bump WORKDIR timestamp, in case tar gave it a timestamp
			# that will interfere with distfiles / WORKDIR timestamp
			# comparisons as reported in bug #332217. Also, fix
			# ownership since tar can change that too.
			os.utime(settings["WORKDIR"], None)
			_prepare_workdir(settings)
		elif self.phase == "install":
			out = io.StringIO()
			_post_src_install_write_metadata(settings)
			_post_src_install_uid_fix(settings, out)
			msg = out.getvalue()
			if msg:
				self.scheduler.output(msg, log_path=logfile)
		elif self.phase == "preinst":
			_preinst_bsdflags(settings)
		elif self.phase == "postinst":
			_postinst_bsdflags(settings)

		post_phase_cmds = _post_phase_cmds.get(self.phase)
		if post_phase_cmds is not None:
			if logfile is not None and self.phase in ("install",):
				# Log to a temporary file, since the code we are running
				# reads PORTAGE_LOG_FILE for QA checks, and we want to
				# avoid annoying "gzip: unexpected end of file" messages
				# when FEATURES=compress-build-logs is enabled.
				fd, logfile = tempfile.mkstemp()
				os.close(fd)
			post_phase = MiscFunctionsProcess(background=self.background,
				commands=post_phase_cmds, logfile=logfile, phase=self.phase,
				scheduler=self.scheduler, settings=settings)
			self._start_task(post_phase, self._post_phase_exit)
			return

		# this point is not reachable if there was a failure and
		# we returned for die_hooks above, so returncode must
		# indicate success (especially if ebuild_process.returncode
		# is unsuccessful and test-fail-continue came into play)
		self.returncode = os.EX_OK
		self._current_task = None
		self.wait()

	def _post_phase_exit(self, post_phase):

		self._assert_current(post_phase)

		log_path = None
		if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
			log_path = self.settings.get("PORTAGE_LOG_FILE")

		if post_phase.logfile is not None and \
			post_phase.logfile != log_path:
			# We were logging to a temp file (see above), so append
			# temp file to main log and remove temp file.
			self._append_temp_log(post_phase.logfile, log_path)

		if self._final_exit(post_phase) != os.EX_OK:
			writemsg("!!! post %s failed; exiting.\n" % self.phase,
				noiselevel=-1)
			self._die_hooks()
			return

		if self.phase == "install":
			out = io.StringIO()
			_post_src_install_soname_symlinks(self.settings, out)
			msg = out.getvalue()
			if msg:
				self.scheduler.output(msg, log_path=log_path)

		self._current_task = None
		self.wait()
		return

	def _append_temp_log(self, temp_log, log_path):

		temp_file = open(_unicode_encode(temp_log,
			encoding=_encodings['fs'], errors='strict'), 'rb')

		log_file, log_file_real = self._open_log(log_path)

		for line in temp_file:
			log_file.write(line)

		temp_file.close()
		log_file.close()
		if log_file_real is not log_file:
			log_file_real.close()
		os.unlink(temp_log)

	def _open_log(self, log_path):

		f = open(_unicode_encode(log_path,
			encoding=_encodings['fs'], errors='strict'),
			mode='ab')
		f_real = f

		if log_path.endswith('.gz'):
			f =  gzip.GzipFile(filename='', mode='ab', fileobj=f)

		return (f, f_real)

	def _die_hooks(self):
		self.returncode = None
		phase = 'die_hooks'
		die_hooks = MiscFunctionsProcess(background=self.background,
			commands=[phase], phase=phase,
			scheduler=self.scheduler, settings=self.settings)
		self._start_task(die_hooks, self._die_hooks_exit)

	def _die_hooks_exit(self, die_hooks):
		if self.phase != 'clean' and \
			'noclean' not in self.settings.features and \
			'fail-clean' in self.settings.features:
			self._default_exit(die_hooks)
			self._fail_clean()
			return
		self._final_exit(die_hooks)
		self.returncode = 1
		self.wait()

	def _fail_clean(self):
		self.returncode = None
		portage.elog.elog_process(self.settings.mycpv, self.settings)
		phase = "clean"
		clean_phase = EbuildPhase(background=self.background,
			phase=phase, scheduler=self.scheduler, settings=self.settings)
		self._start_task(clean_phase, self._fail_clean_exit)
		return

	def _fail_clean_exit(self, clean_phase):
		self._final_exit(clean_phase)
		self.returncode = 1
		self.wait()

	def _elog(self, elog_funcname, lines, background=None):
		if background is None:
			background = self.background
		out = io.StringIO()
		phase = self.phase
		elog_func = getattr(elog_messages, elog_funcname)
		global_havecolor = portage.output.havecolor
		try:
			portage.output.havecolor = \
				self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false')
			for line in lines:


import pprint, sys

import math
import numpy as np
import matplotlib.pyplot as plt
#plt.style.use('bmh_rosh')
from mpl_toolkits.mplot3d import Axes3D
import scipy.stats
from matplotlib.colors import ListedColormap, NoNorm
from matplotlib import mlab
from itertools import cycle # for automatic markers
import matplotlib.patches as patches

from matplotlib.font_manager import FontProperties
from matplotlib.ticker import FuncFormatter
import matplotlib

import matplotlib as mpl
import matplotlib.cm as cm

## local imports
import Task
import libBuffer.Buffer
from SimParams import SimParams
from TaskSet import TaskSet

class Workflow:
    def __init__(self, env, stream_id, stream_resolution, stream_fr, stream_gop_struct):

        self.env = env
        # one-to-one mapping between workflows and inputbuffers
        self.stream_content = None
        self.stream_content_backup = None

        self.stream_id = stream_id
        self.stream_resolution = stream_resolution   # tuple
        self.stream_fr = stream_fr   # framerate
        self.stream_gop_struct = stream_gop_struct

    def populateWorkflow(self):
        # e.g. 1 hr movie, 25fps, 12 gop len = running_time * fps / average_GOP_size
        taskset = TaskSet(self.env)

        task_start_id = self.stream_id * (SimParams.NUM_GOPS * SimParams.GOP_LENGTH)
        gop_start_id = (SimParams.NUM_GOPS * self.stream_id)

        taskset.generateMPEG2FrameTaskSet(SimParams.NUM_GOPS, task_start_id , gop_start_id,
                                          frame_w=self.stream_resolution[0],
                                          frame_h=self.stream_resolution[1])

        # set the worst-case exuction time for all tasks in the task_pool
        taskset.set_worstCaseComputationTime_alltasks()

        self.stream_content = list(taskset.taskList)
        self.stream_content_backup = list(taskset.taskList)

        return len(self.stream_content)

    def removeTask(self, task, ix):
        del self.stream_content[ix]
    def isEmpty(self):
        if(len(self.stream_content)> 0): return False
        else: return True

    def tasksNotDispatched(self, empty_status):
        if(empty_status == True):
            return 0
        else:
            count = 0
            for each_task in  self.stream_content:
                if(each_task.get_dispatchTime() == None) and (self.env.now <= each_task.get_scheduledDispatchTime()) :
                    count += 1
                    break
            return count

    # calculate the total tiles in each VS, each Frame
    def getTotalTiles(self):
        num_tiles = 0
        if SimParams.HEVC_TILELEVEL_SPLITTING_ENABLE == True:
            for each_task in self.stream_content:
                num_tiles += each_task.getNumSubTasksTiles()
            return num_tiles
        else:
            return num_tiles

    ## getters ##
    def get_Task(self):
        return self.stream_content.pop(0)    # return first item
    def get_stream_content(self):
        return self.stream_content

    ## setters ##
    def set_stream_content(self, sc):
        self.stream_content = sc
        self.stream_content_backup = sc

    #############################
    # Standalone functions
    #############################
    @staticmethod
    def dumpWorkflowsToFile(all_workflows, fname="workflows.xml"):
        file = open(fname, "w")
        file.write("<Workflows>")
        for each_workflow in all_workflows:
            file.write("<TaskSet>")
            for each_task in each_workflow.stream_content :
                file.write( each_task._debugLongXML() )
                file.write("\n")
            file.write("</TaskSet>")
        file.write("</Workflows>")
        file.close()

    @staticmethod
    def plot_TaskComputationCostHistogram(list_workflows, fig_id, ftype="all", wf_ids = range(SimParams.NUM_WORKFLOWS)):
        print "plot_TaskComputationCostHistogram:: Enter"
        nrows = int(math.ceil(len(wf_ids) / 2.))
        print nrows
        if(nrows<2): nrows=2
        fig = plt.figure(fig_id+100)

        for ix,each_wf_id in enumerate(wf_ids):

            print ix

            hist_data = []
            for each_task in list_workflows[each_wf_id].stream_content_backup:
                if(ftype == "all"):
                    hist_data.append(each_task.get_computationCost())
                else:
                    if each_task.get_frameType() == ftype:
                        hist_data.append(each_task.get_computationCost())

            print len(hist_data)

            print "unique values : ", len(set(hist_data))
            print "max_val : ", np.max(hist_data)
            print "min_val : ", np.min(hist_data)

            plt.subplot(nrows, 2, ix+1)
            fig.canvas.set_window_title('TaskComputationCostHistogram - ' + ftype)
            count, bins, ignored = plt.hist(hist_data, alpha=0.5, color='g', bins=int(len(list_workflows[each_wf_id].stream_content_backup)/1), normed=1)
            plt.plot(bins, np.ones_like(bins), linewidth=2, color='r')
            plt.grid(True)
            print "finished generating plot"

    @staticmethod
    # step plot with all types of frames overlapping histograms
    # assumes single wf ==> single vid
    def plot_TaskComputationCostHistogram_step(list_workflows, fig_id, wf_ids = range(SimParams.NUM_WORKFLOWS)):

        def to_percent(y, position):
            # Ignore the passed in position. This has the effect of scaling the default
            # tick locations.
            s = str(100 * y)
            # The percent symbol needs escaping in latex
            if matplotlib.rcParams['text.usetex'] is True:
                return s + r'$\%$'
            else:
                return s + '%'

        temp_cols = ['#FF8585', '#85FF85', '#8585FF']
        temp_cols = ['r','g','b']

        print "plot_TaskComputationCostHistogram:: Enter"
        f, axarr = plt.subplots(len(wf_ids))
        f.canvas.set_window_title('TaskComputationCostHistogram')

        for ix,each_wf_id in enumerate(wf_ids):
            print ix
            vid_res = "%dx%d" % (list_workflows[each_wf_id].stream_content_backup[0].get_frame_w(),
                            list_workflows[each_wf_id].stream_content_backup[0].get_frame_h()
                            )

            I_frame_compcost_dist = []
            P_frame_compcost_dist = []
            B_frame_compcost_dist = []

            for each_task in list_workflows[each_wf_id].stream_content_backup:
                ftype = each_task.get_frameType()
                if (ftype == "I") : I_frame_compcost_dist.append(each_task.get_computationCost())
                elif (ftype == "P") : P_frame_compcost_dist.append(each_task.get_computationCost())
                elif (ftype == "B") : B_frame_compcost_dist.append(each_task.get_computationCost())
                else: sys.exit("undefined frame type")

            print "I_frame_compcost_dist",len(I_frame_compcost_dist), np.max(I_frame_compcost_dist), np.min(I_frame_compcost_dist)
            print "P_frame_compcost_dist",len(P_frame_compcost_dist), np.max(P_frame_compcost_dist), np.min(P_frame_compcost_dist)
            print "B_frame_compcost_dist",len(B_frame_compcost_dist), np.max(B_frame_compcost_dist), np.min(B_frame_compcost_dist)

            all_ftypes_data = [I_frame_compcost_dist, P_frame_compcost_dist, B_frame_compcost_dist]

#             weights = [np.ones_like(I_frame_compcost_dist)/float(len(I_frame_compcost_dist)),
#                        np.ones_like(P_frame_compcost_dist)/float(len(P_frame_compcost_dist)),
#                        np.ones_like(B_frame_compcost_dist)/float(len(B_frame_compcost_dist))]

            weights = [np.zeros_like(I_frame_compcost_dist) + 1. / float(len(I_frame_compcost_dist)),
                       np.zeros_like(P_frame_compcost_dist) + 1. / float(len(P_frame_compcost_dist)),
                       np.zeros_like(B_frame_compcost_dist) + 1. / float(len(B_frame_compcost_dist))]

            count, bins, ignored = axarr[ix].hist(all_ftypes_data,
                                                #alpha=1.0,
                                                #color=['r','g','b'],
                                                color=temp_cols,
                                                linewidth=2.0,
                                                histtype='step', fill=False,
                                                bins=100, weights=weights)

            #axarr[ix].set_title(vid_res)

#             axarr[ix].text(.5,.9,'res: %s' %(vid_res),
#                             horizontalalignment='center',
#                             transform=axarr[ix].transAxes,
#                             fontsize=12)
#

            formatter = FuncFormatter(to_percent)
            axarr[ix].yaxis.set_major_formatter(formatter)

            axarr[ix].grid(True)

            #axarr[ix].set_ylabel("Normalised frequency (%)")

            rect_lbl_list = ["I-frame", "P-frame", "B-frame"]
            rec_r = patches.Rectangle( (0.72, 0.1), 0.2, 0.6, facecolor=temp_cols[0] )
            rec_g = patches.Rectangle( (0.72, 0.1), 0.2, 0.6, facecolor=temp_cols[1] )
            rec_b = patches.Rectangle( (0.72, 0.1), 0.2, 0.6, facecolor=temp_cols[2] )

            rects_list = [rec_r, rec_g, rec_b]

            leg = axarr[ix].legend(rects_list,
                            rect_lbl_list,
                            ncol=1,
                            prop={'size':12},
                            loc='right center',
                            bbox_to_anchor=(0.5, 1.05),
                            title='res: %s' %(vid_res)
                            )
            leg.draggable()
            axarr[ix].get_legend().get_title().set_fontsize('12')

            print "finished generating plot"

        axarr[-1].set_xlabel("Frame decoding time (s)")
        #f.text(0.04, 0.5, "Normalised frequency (%)", va='center', rotation='vertical', fontsize=12, color='#5d5d5d') # for ggplot
        f.text(0.01, 0.5, "Normalised frequency (%)", va='center', rotation='vertical', fontsize=14)

    @staticmethod
    def plot_TaskComputationCost(list_workflows, fig_id, ftype="all", wf_ids = range(SimParams.NUM_WORKFLOWS)):

        nrows = int(math.ceil(len(wf_ids) / 2.))
        print nrows
        if(nrows<2): nrows=2
        #fig, axs = plt.subplots(nrows, 2)
        fig = plt.figure(fig_id+100)

        # color based on video stream id
        #norm = mpl.colors.Normalize(vmin=0, vmax=SimParams.WFGEN_MAX_VIDS_PER_WF)
        cmap = mpl.cm.get_cmap('gist_rainbow')

        for ix,each_wf_id in enumerate(wf_ids):

            compcost = []
            task_ids = []
            stream_specific_cols = []

            for each_task in list_workflows[each_wf_id].stream_content_backup:


#
# Copyright (c) 2010 - 2012 Red Hat, Inc.
#
# This software is licensed to you under the GNU General Public License,
# version 2 (GPLv2). There is NO WARRANTY for this software, express or
# implied, including the implied warranties of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2
# along with this software; if not, see
# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt.
#
# Red Hat trademarks are not licensed under GPLv2. No permission is
# granted to use or replicate Red Hat trademarks that are incorporated
# in this software or its documentation.
#

from cStringIO import StringIO
import errno
import gettext
import os
import sys
from zipfile import ZipFile, BadZipfile

from rhsm import certificate

from rct.commands import RCTCliCommand
from rct.printing import xstr
from subscription_manager.cli import InvalidCLIOptionError
from rhsm import ourjson as json

_ = gettext.gettext

def get_value(json_dict, path):
    current = json_dict
    for item in path.split("."):
        if item in current:
            current = current[item]
        else:
            return ""

    return current

class ZipExtractAll(ZipFile):
    """extend ZipFile with a safer extractall

    Zipfile() does not support extractall on python2.4, and the 2.6 versions
    are known to be unsafe in how they extract files. 2.6 version does not
    validate that files are within the archive root, or check that files are
    created safely.

    Contains helper methods for manipulating and reading
    the zipfile more easily in memory"""

    inner_zip = None

    def __init__(self, *args, **kwargs):
        """
        Validates the zip file
        """
        try:
            ZipFile.__init__(self, *args, **kwargs)
        except BadZipfile:
            print _("Manifest zip is invalid.")
            sys.exit(1)

    def _get_inner_zip(self):
        if self.inner_zip is None:
            output = StringIO(self.read(RCTManifestCommand.INNER_FILE))
            self.inner_zip = ZipExtractAll(output, 'r')
        return self.inner_zip

    def _read_file(self, file_path, is_inner=False):
        try:
            output = StringIO(self.read(file_path))
            result = output.getvalue()
            output.close()
        except KeyError:
            try:
                if is_inner:
                    raise KeyError
                result = self._get_inner_zip()._read_file(file_path, True)
            except KeyError:
                raise Exception(_('Unable to find file "%s" in manifest.') % file_path)
        return result

    def _get_entitlements(self):
        results = []
        in_zip = self._get_inner_zip()
        for filename in in_zip.namelist():
            (read_path, read_file) = os.path.split(filename)
            if (read_path == os.path.join("export", "entitlements")) and (len(read_file) > 0):
                results.append(filename)
        return results

    def _open_excl(self, path):
        return os.fdopen(os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL), 'w')

    def _write_file(self, output_path, archive_path):
        outfile = self._open_excl(output_path)
        outfile.write(self.read(archive_path))
        outfile.close()

    def _is_secure(self, base, new_file):
        base_path = os.path.abspath(base)
        new_path = os.path.abspath(new_file)
        if not new_path.startswith(base_path):
            raise Exception(_('Manifest zip attempted to extract outside of the base directory.'))
        #traces symlink to source, and checks that it is valid
        real_new_path = os.path.realpath(new_path)
        if real_new_path != new_path:
            self._is_secure(base, real_new_path)
        elif os.path.islink(new_path):
            raise Exception(_('Unable to trace symbolic link.  Possibly circular linkage.'))

    def extractall(self, location, overwrite=False):
        self._is_secure(location, location)
        for path_name in self.namelist():
            (directory, filename) = os.path.split(path_name)
            directory = os.path.join(location, directory)
            self._is_secure(location, directory)
            if not os.path.exists(directory):
                os.makedirs(directory)
            new_location = os.path.join(directory, filename)
            self._is_secure(location, new_location)
            if (os.path.exists(new_location) and overwrite):
                os.remove(new_location)
            self._write_file(new_location, path_name)

class RCTManifestCommand(RCTCliCommand):

    INNER_FILE = "consumer_export.zip"

    def __init__(self, name="cli", aliases=None, shortdesc=None, primary=False):
        RCTCliCommand.__init__(self, name=name, aliases=aliases,
                shortdesc=shortdesc, primary=primary)

    def _get_usage(self):
        return _("%%prog %s [OPTIONS] MANIFEST_FILE") % self.name

    def _validate_options(self):
        manifest_file = self._get_file_from_args()
        if not manifest_file:
            raise InvalidCLIOptionError(_("You must specify a manifest file."))

        if not os.path.isfile(manifest_file):
            raise InvalidCLIOptionError(_("The specified manifest file does not exist."))

    def _extract_manifest(self, location, overwrite=False):
        # Extract the outer file
        archive = ZipExtractAll(self._get_file_from_args(), 'r')
        archive.extractall(location, overwrite)

        # now extract the inner file
        if location:
            inner_file = os.path.join(location, self.INNER_FILE)
        else:
            inner_file = self.INNER_FILE

        archive = ZipExtractAll(inner_file, 'r')
        archive.extractall(location, overwrite)

        # Delete the intermediate file
        os.remove(inner_file)

class CatManifestCommand(RCTManifestCommand):

    def __init__(self):
        RCTManifestCommand.__init__(self, name="cat-manifest", aliases=['cm'],
                               shortdesc=_("Print manifest information"),
                               primary=True)

    def _print_section(self, title, items, indent=1, whitespace=True):
        # Allow a bit of customization of the tabbing
        pad = "\t" * (indent - 1)
        print(pad + title)
        pad += "\t"
        for item in items:
            if len(item) == 2:
                print("%s%s: %s" % (pad, item[0], xstr(item[1])))
            else:
                print("%s%s" % (pad, item[0]))
        if whitespace:
            print ""

    def _print_general(self, zip_archive):
        # Print out general data
        part = zip_archive._read_file(os.path.join("export", "meta.json"))
        data = json.loads(part)
        to_print = []
        to_print.append((_("Server"), get_value(data, "webAppPrefix")))
        to_print.append((_("Server Version"), get_value(data, "version")))
        to_print.append((_("Date Created"), get_value(data, "created")))
        to_print.append((_("Creator"), get_value(data, "principalName")))
        self._print_section(_("General:"), to_print)

    def _print_consumer(self, zip_archive):
        # Print out the consumer data
        part = zip_archive._read_file(os.path.join("export", "consumer.json"))
        data = json.loads(part)
        to_print = []
        to_print.append((_("Name"), get_value(data, "name")))
        to_print.append((_("UUID"), get_value(data, "uuid")))
        to_print.append((_("Type"), get_value(data, "type.label")))
        self._print_section(_("Consumer:"), to_print)

    def _get_product_attribute(self, name, data):
        return_value = None
        for attr in get_value(data, "pool.productAttributes"):


#!/usr/bin/env python

# This file is part of VoltDB.
# Copyright (C) 2008-2013 VoltDB Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.

"""
VoltDB catalog code generator.
"""

from catalog_utils import *
from string import Template
from subprocess import Popen

#
# Code generation (shared).
#

def writer( f ):
    def write( *args ):
        f.write( ' '.join( map( str, args ) ) + '\n' )
    return write

def interp(text, params = locals()):
    t = Template(text)
    #return t.safe_substitute(params)
    return t.substitute(params)

#
# Java code generation.
#

def javatypify( x ):
    if x == 'string': return 'String'
    elif x == 'int': return 'int'
    elif x == 'bool': return 'boolean'
    elif x[-1] == '*': return 'CatalogMap<%s>' % x.rstrip('*')
    elif x[-1] == '?': return x.rstrip('?')
    else: raise Exception( 'bad type: ' + x )

def javaobjectify( x ):
    if x == 'string': return 'String'
    elif x == 'int': return 'Integer'
    elif x == 'bool': return 'Boolean'
    elif x[-1] == '*': return 'CatalogMap<%s>' % x.rstrip('*')
    elif x[-1] == '?': return x.rstrip('?')
    else: raise Exception( 'bad type: ' + x )

def genjava( classes, prepath, postpath, package ):
    ##########
    # SETUP
    ##########
    pkgdir = package.replace('.', '/')
    os.system( interp( "rm -rf $postpath/*", locals() ) )
    os.system( interp( "mkdir -p $postpath/", locals() ) )
    os.system( interp( "cp $prepath/Catalog.java $postpath", locals() ) )
    os.system( interp( "cp $prepath/CatalogType.java $postpath", locals() ) )
    os.system( interp( "cp $prepath/CatalogMap.java $postpath", locals() ) )
    os.system( interp( "cp $prepath/CatalogException.java $postpath", locals() ) )

    ##########
    # WRITE THE SOURCE FILES
    ##########

    for cls in classes:
        clsname = cls.name
        javapath = postpath + "/" + clsname + '.java'
        #ensure_relative_path_exists(postpath + "/" + pkgdir)
        f = file( javapath, 'w' )
        if not f:
            raise OSError("Can't create file %s for writing" % javapath)
        write = writer( f )
        write (gpl_header)
        write (auto_gen_warning)
        write('package', package + ';\n')

        if cls.has_comment():
            write('/**\n *', cls.comment)
            write(' */')

        write( interp( 'public class $clsname extends CatalogType {\n', locals() ) )

        # fields
        for field in cls.fields:
            ftype = javatypify( field.type )
            fname = field.name
            #realtype = field.type[:-1]
            #methname = fname.capitalize()
            if ftype == "String":
                write( interp( '    String m_$fname = new String();', locals() ) )
            elif field.type[-1] == '?':
                pass # don't keep local cached vars for references
            else:
                write( interp( '    $ftype m_$fname;', locals() ) )
        write( '' )

        # setBaseValues
        write( '    void setBaseValues(Catalog catalog, CatalogType parent, String path, String name) {' )
        write( '        super.setBaseValues(catalog, parent, path, name);')
        for field in cls.fields:
            ftype = javatypify( field.type )
            fname = field.name
            realtype = field.type[:-1]
            #methname = fname.capitalize()
            if field.type[-1] == '*':
                write( interp( '        m_$fname = new $ftype(catalog, this, path + "/" + "$fname", $realtype.class);', locals() ) )
                write( interp( '        m_childCollections.put("$fname", m_$fname);', locals() ) )
            elif field.type[-1] == '?':
                write( interp( '        m_fields.put("$fname", null);', locals() ) )
            else:
                write( interp( '        m_fields.put("$fname", m_$fname);', locals() ) )
        write( '    }\n' )

        # update
        write ( '    void update() {' )
        for field in cls.fields:
            ftype = javatypify( field.type )
            fobjtype = javaobjectify( field.type )
            fname = field.name
            realtype = field.type[:-1]
            methname = fname.capitalize()
            if field.type[-1] == '?':
                pass # don't keep local cached vars for references
            elif field.type[-1] != '*':
                write( interp( '        m_$fname = ($fobjtype) m_fields.get("$fname");', locals() ) )
        write( '    }\n' )

        # getter methods
        for field in cls.fields:
            ftype = javatypify( field.type )
            fname = field.name
            realtype = field.type[:-1]
            methname = fname.capitalize()
            if field.has_comment():
                write('    /** GETTER:', field.comment, '*/')
            write( interp( '    public $ftype get$methname() {', locals() ) )
            if field.type[-1] == '?':
                write( interp( '        Object o = getField("$fname");', locals() ) )
                write( interp( '        if (o instanceof UnresolvedInfo) {', locals() ) )
                write( interp( '            UnresolvedInfo ui = (UnresolvedInfo) o;', locals() ) )
                write( interp( '            $ftype retval = ($ftype) m_catalog.getItemForRef(ui.path);', locals() ) )
                write( interp( '            assert(retval != null);', locals() ) )
                write( interp( '            m_fields.put("$fname", retval);', locals() ) )
                write( interp( '            return retval;', locals() ) )
                write( interp( '        }', locals() ) )
                write( interp( '        return ($ftype) o;', locals() ) )
            else:
                write( interp( '        return m_$fname;', locals() ) )
            write( '    }\n' )

        # setter methods
        for field in cls.fields:
            ftype = javatypify( field.type )
            fname = field.name
            realtype = field.type[:-1]
            methname = fname.capitalize()
            if field.type[-1] == '*':
                continue
            if field.has_comment():
                write('    /** SETTER:', field.comment, '*/')
            write( interp( '    public void set$methname($ftype value) {', locals() ) )
            if field.type[-1] == '?':
                write( interp( '        m_fields.put("$fname", value);', locals() ) )
            else:
                write( interp( '        m_$fname = value; m_fields.put("$fname", value);', locals() ) )
            write( '    }\n' )

        # wrap up
        write( '}' )

#
# C++ code generation.
#

def cpptypify( x ):
    if x == 'string': return 'std::string'
    elif x == 'int': return 'int32_t'
    elif x == 'bool': return 'bool'
    elif x[-1] == '*': return 'CatalogMap<%s>' % x.rstrip('*')
    elif x[-1] == '?': return 'CatalogType*'
    else: raise Exception( 'bad type: ' + x )

def gencpp( classes, prepath, postpath ):
    ##########
    # SETUP
    ##########
    os.system( interp( "rm -rf $postpath/*", locals() ) )
    os.system( interp( "mkdir -p $postpath/", locals() ) )
    os.system( interp( "cp $prepath/catalog.h $postpath", locals() ) )
    os.system( interp( "cp $prepath/catalogtype.h $postpath", locals() ) )
    os.system( interp( "cp $prepath/catalogmap.h $postpath", locals() ) )
    os.system( interp( "cp $prepath/catalog.cpp $postpath", locals() ) )
    os.system( interp( "cp $prepath/catalogtype.cpp $postpath", locals() ) )

    ##########
    # WRITE THE SOURCE FILES
    ##########
    for cls in reversed( classes ):
        clsname = cls.name

        referencedClasses = []
        for field in cls.fields:
            classType = field.type[:-1]
            if (field.type[-1] == "*") or (field.type[-1] == '?'):
                if classType not in referencedClasses:
                    referencedClasses.append(classType)
        if cls.name in referencedClasses:
            referencedClasses.remove(cls.name)

        ##########
        # WRITE THE HEADER FILE
        ##########

        f = file( postpath + "/" + clsname.lower() + ".h", 'w' )
        write = writer( f )

        write (gpl_header)
        write (auto_gen_warning)
        pp_unique_str = "CATALOG_" + clsname.upper() + "_H_"
        write("#ifndef", pp_unique_str);
        write("#define", pp_unique_str);
        write("")

        write('#include <string>')
        write('#include "catalogtype.h"')
        write('#include "catalogmap.h"\n')
        write('namespace catalog {\n')

        for classType in referencedClasses:
            write("class " + classType + ";")

        if cls.has_comment():
            write('/**\n *', cls.comment)
            write(' */')

        write( interp( 'class $clsname : public CatalogType {', locals() ) )
        write( '    friend class Catalog;' )
        write( interp( '    friend class CatalogMap<$clsname>;', locals() ) )

        # protected section
        write( '\nprotected:')

        # constructor
        write( '    ' + clsname + '(Catalog * catalog, CatalogType * parent, const std::string &path, const std::string &name);' )

        # Field Member variables.
        for field in cls.fields:
            ftype = cpptypify( field.type )
            privname = 'm_' + field.name
            write( interp( '    $ftype $privname;', locals() ) )

        # update method
        write("\n    virtual void update();\n")

        # add method
        write("    virtual CatalogType * addChild(const std::string &collectionName, const std::string &name);")

        # getChild method
        write("    virtual CatalogType * getChild(const std::string &collectionName, const std::string &childName) const;")

        # removeChild method
        write("    virtual bool removeChild(const std::string &collectionName, const std::string &childName);")

        # public section
        write("\npublic:")

        # destructor
        write('    ~' + clsname + '();\n');

        # getter methods
        for field in cls.fields:
            ftype = cpptypify( field.type )
            privname = 'm_' + field.name
            pubname = field.name
            Pubname = pubname.capitalize()
            if field.has_comment():
                write('    /** GETTER:', field.comment, '*/')
            if field.type == 'string':
                write ( interp( '    const std::string & $pubname() const;', locals() ) )
            elif field.type[-1] == '?':
                write ( "    const " + field.type[:-1] + " * " + pubname + "() const;")
            elif field.type[-1] == '*':
                write ( interp( '    const $ftype & $pubname() const;', locals() ) )
            else:
                write ( interp( '    $ftype $pubname() const;', locals() ) )

        write( '};\n' )

        write( '} // namespace catalog\n' )

        write ("#endif // ", pp_unique_str)

        ##########
        # WRITE THE CPP FILE
        ##########

        f = file( postpath + "/" + clsname.lower() + ".cpp", 'w' )
        write = writer( f )

        write (gpl_header)
        write (auto_gen_warning)
        filename = clsname.lower()
        write ( '#include <cassert>' )
        write ( interp( '#include "$filename.h"', locals() ) )
        write ( '#include "catalog.h"' )
        otherhdrs = ['#include "%s.h"' % field.type[:-1].lower() for field in cls.fields if field.type[-1] in ['*', '?'] ]
        uniques = {}
        for hdr in otherhdrs:
            uniques[hdr] = hdr
        for hdr in uniques.keys():
            write( hdr )
        write ( '\nusing namespace catalog;' )
        write ( 'using namespace std;\n' )

        # write the constructor
        mapcons = ["m_%s(catalog)" % field.name for field in cls.fields if field.type[-1] == '*']
        write ( interp( '$clsname::$clsname(Catalog *catalog, CatalogType *parent, const string &path, const string &name)', locals() ) )
        comma = ''
        if len(mapcons): comma = ','
        write ( interp( ': CatalogType(catalog, parent, path, name)$comma', locals()))

        mapcons = ["m_%s(catalog, this, path + \"/\" + \"%s\")" % (field.name, field.name) for field in cls.fields if field.type[-1] == '*']
        if len(mapcons) > 0:
            write( "  " + ", ".join(mapcons))
        write('{')

        # init the fields and childCollections
        write( '    CatalogValue value;' )
        for field in cls.fields:
            ftype = cpptypify( field.type )
            privname = 'm_' + field.name
            pubname = field.name


from unittest import TestCase

from Crypto.PublicKey import RSA
from lib.xecrypt import XeCryptBnQw_SwapLeBe, XeCryptBnDw_SwapLeBe, XeCryptBnQw, XeCryptBnQw_toInt, XeCryptBnQwNeMod
from lib.xecrypt_rsa import XeCrypt_RSA

from lib.xecrypt import XeCryptRotSum, RotSumCtx
from lib.xecrypt import XeCryptRotSumSha

class TestXeCryptRotSumSha(TestCase):

    def test_normal(self):
        buffer1 = bytearray(0x40)
        for i in range(len(buffer1)):
            buffer1[i] = i % 255

        buffer2 = bytearray(0x40)
        for i in range(len(buffer2)):
            buffer2[i] = 255 - (i % 255)

        digest = bytes(
            [0x31, 0xf0, 0x0b, 0x77, 0x68, 0xf9, 0x57, 0x71, 0x8c, 0x4d, 0x62, 0x84, 0xb3, 0x45, 0xcd, 0xdd, 0x37, 0x9c,
             0x6e, 0xe9])
        self.assertEqual(XeCryptRotSumSha(buffer1, buffer2, 0x20), digest)

    def test_input_2_zero(self):
        buffer1 = bytearray(0x40)
        for i in range(len(buffer1)):
            buffer1[i] = i % 255

        buffer2 = bytearray(0x40)

        digest = bytes(
            [0x2a, 0xdb, 0xc2, 0x8c, 0xda, 0xca, 0x48, 0x8e, 0xa5, 0x01, 0x74, 0xa5, 0xd9, 0x80, 0x60, 0xc5, 0xf9, 0x0f,
             0x5d, 0x54])
        self.assertEqual(XeCryptRotSumSha(buffer1, buffer2, 0x20), digest)

    def test_input_all_zero(self):
        buffer1 = bytearray(0x40)
        buffer2 = bytearray(0x40)

        digest = bytes(
            [0x37, 0xd3, 0xb6, 0xb8, 0x4e, 0x35, 0x08, 0x0a, 0xe5, 0xf9, 0x60, 0xfd, 0xbf, 0x44, 0x26, 0x06, 0x54, 0x02,
             0x7e, 0x5b])
        self.assertEqual(XeCryptRotSumSha(buffer1, buffer2, 0x20), digest)

class TestXeCryptRotSum(TestCase):

    def test_launch_xex(self):

        input_1_size = 0x70
        input_1 = bytearray(
            [0x00, 0x00, 0x01, 0x74, 0x00, 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, 0x00, 0xfc, 0xa0, 0x61, 0xd4,
             0xc2, 0xa1, 0x71, 0xc0, 0xec, 0x98, 0x00, 0x86, 0xe6, 0x8b, 0xa8, 0x9f, 0xb0, 0x9c, 0x83, 0xd4,
             0x00, 0x00, 0x00, 0x02, 0x47, 0x88, 0x8f, 0x6a, 0x94, 0x36, 0x58, 0xc0, 0x7a, 0xdf, 0xeb, 0xda,
             0x44, 0x13, 0x4a, 0x7f, 0xe0, 0xd2, 0x3f, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
             0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, 0xd8, 0x0b, 0xfa, 0xf3, 0xdc, 0x53, 0xb0,
             0x89, 0x7d, 0x5e, 0xb5, 0x08, 0xd0, 0x10, 0x63, 0x00, 0x00, 0x00, 0x00, 0xca, 0xe2, 0xc7, 0x20,
             0xef, 0xb2, 0x9a, 0x1d, 0x2e, 0x90, 0xe4, 0x7e, 0xb9, 0xdf, 0x41, 0xa0, 0xf5, 0xf5, 0xb7, 0xa4])

        rotsum_ctx = RotSumCtx([0] * 4)

        output = bytes([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0xb2, 0x51, 0xdc, 0xa2, 0xc4, 0x60, 0xa3, 0x5d,
                        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf7, 0xd8, 0x2d, 0xc6, 0x4e, 0xec, 0xd3, 0xea, 0x24])

        XeCryptRotSum(rotsum_ctx, input_1, input_1_size >> 3)

        self.assertEqual(bytes(rotsum_ctx), output)

    def test_launch_xex_zero_all(self):

        input_1_size = 0x0
        input_1 = []

        output = bytes([0x0] * 0x20)

        rotsum_ctx = RotSumCtx.from_bytes(output)

        XeCryptRotSum(rotsum_ctx, input_1, input_1_size >> 3)

        self.assertEqual(bytes(rotsum_ctx), output)

    def test_launch_xex_zero_input(self):

        input_1_size = 0x0
        input_1 = []

        output = bytes([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0xb2, 0x51, 0xdc, 0xa2, 0xc4, 0x60, 0xa3, 0x5d,
                        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf7, 0xd8, 0x2d, 0xc6, 0x4e, 0xec, 0xd3, 0xea, 0x24])

        rotsum_ctx = RotSumCtx.from_bytes(output)

        XeCryptRotSum(rotsum_ctx, input_1, input_1_size >> 3)

        self.assertEqual(bytes(rotsum_ctx), output)

    def test_normal(self):

        input_1_size = 0x40

        input_1_bytes = bytearray(
            [0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
             0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
             0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
             0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f])

        input_1 = bytearray(input_1_size)
        for i in range(input_1_size):
            input_1[i] = i % 255

        self.assertEqual(input_1_bytes, input_1)

        rotsum_ctx = RotSumCtx([0] * 4)

        output = bytes([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x83, 0x62, 0xe0, 0xd8, 0xd7, 0xd3, 0xc3, 0x83,
                        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfb, 0x6d, 0x14, 0x14, 0x14, 0x19, 0x69, 0x69, 0x69])

        XeCryptRotSum(rotsum_ctx, input_1, input_1_size >> 3)

        self.assertEqual(bytes(rotsum_ctx), output)

    def test_normal_2(self):

        input_2_size = 0x40

        input_2_bytes = bytearray(
            [0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, 0xf7, 0xf6, 0xf5, 0xf4, 0xf3, 0xf2, 0xf1, 0xf0,
             0xef, 0xee, 0xed, 0xec, 0xeb, 0xea, 0xe9, 0xe8, 0xe7, 0xe6, 0xe5, 0xe4, 0xe3, 0xe2, 0xe1, 0xe0,
             0xdf, 0xde, 0xdd, 0xdc, 0xdb, 0xda, 0xd9, 0xd8, 0xd7, 0xd6, 0xd5, 0xd4, 0xd3, 0xd2, 0xd1, 0xd0,
             0xcf, 0xce, 0xcd, 0xcc, 0xcb, 0xca, 0xc9, 0xc8, 0xc7, 0xc6, 0xc5, 0xc4, 0xc3, 0xc2, 0xc1, 0xc0])

        input_2 = bytearray(input_2_size)
        for i in range(input_2_size):
            input_2[i] = 255 - (i % 255)

        self.assertEqual(input_2_bytes, input_2)

        output_1 = bytes([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
                          0x83, 0x62, 0xe0, 0xd8, 0xd7, 0xd3, 0xc3, 0x83,
                          0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfb,
                          0x6d, 0x14, 0x14, 0x14, 0x19, 0x69, 0x69, 0x69])

        rotsum_ctx = RotSumCtx.from_bytes(output_1)

        output_2 = bytes([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06,
                          0x50, 0x60, 0x61, 0xaa, 0x6a, 0x8b, 0x0d, 0x54,
                          0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8,
                          0x52, 0x59, 0x00, 0x00, 0xa4, 0xb0, 0x00, 0x00])

        XeCryptRotSum(rotsum_ctx, input_2, input_2_size >> 3)

        self.assertEqual(bytes(rotsum_ctx), output_2)

class TestXeCryptBnQw_SwapLeBe(TestCase):

    def test_normal_1(self):
        self.assertEqual(XeCryptBnQw_SwapLeBe(
            bytes([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]), 1),
            bytes([0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01]))

    def test_normal_2(self):
        self.assertEqual(XeCryptBnQw_SwapLeBe(
            bytes([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
                   0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
                   0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]), 3),
            bytes([0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01,
                   0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01,
                   0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01]))

    def test_zero(self):
        self.assertEqual(XeCryptBnQw_SwapLeBe(
            bytes([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]), 1),
            bytes([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]))

    def test_one(self):
        self.assertEqual(XeCryptBnQw_SwapLeBe(
        bytes([0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]), 1),
        bytes([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01]))

class TestXeCryptBnDw_SwapLeBe(TestCase):

    def test_normal_1(self):
        self.assertEqual(XeCryptBnDw_SwapLeBe(
            bytes([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]), 2),
            bytes([0x04, 0x03, 0x02, 0x01, 0x08, 0x07, 0x06, 0x05, ]))

    def test_normal_2(self):
        self.assertEqual(XeCryptBnDw_SwapLeBe(
            bytes([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
                   0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
                   0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]), 6),
            bytes([0x04, 0x03, 0x02, 0x01, 0x08, 0x07, 0x06, 0x05,
                   0x04, 0x03, 0x02, 0x01, 0x08, 0x07, 0x06, 0x05,
                   0x04, 0x03, 0x02, 0x01, 0x08, 0x07, 0x06, 0x05]))

    def test_zero(self):
        self.assertEqual(XeCryptBnDw_SwapLeBe(
            bytes([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]), 2),
            bytes([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]))

    def test_one(self):
        self.assertEqual(XeCryptBnDw_SwapLeBe(
            bytes([0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]), 2),
            bytes([0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00]))

class TestXeCryptBnQw(TestCase):

    def test_rsa(self):
        rsa_obj = RSA.generate(1024)

        p_old_bn = XeCryptBnQw(rsa_obj.p, 64)
        self.assertEqual(XeCryptBnQw_toInt(p_old_bn), rsa_obj.p)

        n_old_bn = XeCryptBnQw(rsa_obj.n, 128)
        self.assertEqual(XeCryptBnQw_toInt(n_old_bn), rsa_obj.n)

    def test_xersa(self):
        rsa_obj = RSA.generate(1024)
        xecrypt_obj = XeCrypt_RSA.from_rsa_obj(rsa_obj)

        p_old_bn = XeCryptBnQw(xecrypt_obj.p, 64)
        self.assertEqual(XeCryptBnQw_toInt(p_old_bn), xecrypt_obj.p)

        n_old_bn = XeCryptBnQw(xecrypt_obj.n, 128)
        self.assertEqual(XeCryptBnQw_toInt(n_old_bn), xecrypt_obj.n)

class TestXeCryptBnQwNeMod(TestCase):

    def test_normal(self):
        r3 = bytes([
            0xf5, 0xbf, 0x46, 0xe4, 0x34, 0xf1, 0x73, 0x3c, 0xef, 0x6d, 0x46, 0x91, 0x3d, 0x75, 0x08, 0xcf,
            0x3f, 0x09, 0x57, 0xcd, 0x7e, 0xe5, 0x1a, 0xa9, 0x7d, 0x83, 0x99, 0xf8, 0xf6, 0xe9, 0x58, 0x39,
            0xa5, 0x37, 0xf0, 0x6c, 0x56, 0xa7, 0xef, 0x33, 0xe8, 0x11, 0x23, 0x26, 0x1b, 0x94, 0x9e, 0xc8,
            0xcc, 0x30, 0x81, 0x7b, 0x21, 0x9f, 0x6c, 0x54, 0x39, 0x20, 0x35, 0x88, 0xbb, 0xe1, 0x98, 0xf6,
            0x62, 0x6c, 0xa9, 0x28, 0x65, 0xd4, 0x70, 0xe8, 0x40, 0xb2, 0xd5, 0x3d, 0x94, 0x9d, 0x45, 0x27,
            0x0c, 0xc9, 0xf4, 0x09, 0x24, 0x12, 0x08, 0x59, 0xaf, 0x7c, 0xd2, 0xf3, 0xc4, 0x9a, 0xcd, 0x3d,
            0x6c, 0xe7, 0xe9, 0x6f, 0x22, 0x49, 0x18, 0x1e, 0x40, 0x72, 0x0b, 0x80, 0x47, 0x49, 0x52, 0x5f,
            0x33, 0x70, 0x77, 0xff, 0xbb, 0xc4, 0xf4, 0xb5, 0xdb, 0xbb, 0x3f, 0x02, 0x55, 0x73, 0xe9, 0x34,
            0xa1, 0x99, 0x04, 0xc0, 0x43, 0xef, 0x73, 0x4e, 0xc6, 0xaa, 0xc7, 0xe1, 0x07, 0x92, 0xb7, 0x18,
            0x84, 0x2f, 0x0d, 0x44, 0xa1, 0x5f, 0xe6, 0x7c, 0x15, 0x43, 0xbe, 0xa4, 0x16, 0x15, 0x03, 0xd0,
            0x12, 0x7a, 0x3a, 0xeb, 0xdc, 0xf1, 0x7e, 0x2d, 0x05, 0x5e, 0xb9, 0x21, 0xbe, 0x4e, 0xeb, 0xf9,
            0x78, 0x69, 0x40, 0x7f, 0x38, 0xe8, 0x19, 0xe2, 0x41, 0x45, 0xbf, 0x0c, 0x5f, 0xa8, 0x8c, 0xdc,
            0x1e, 0x52, 0x0e, 0x37, 0x7a, 0x39, 0xd7, 0x6a, 0x5e, 0x65, 0x06, 0x8c, 0x85, 0x5d, 0x0a, 0xc9,
            0xb4, 0xfd, 0xfe, 0x31, 0x6e, 0x8c, 0x0d, 0x7d, 0xa1, 0xad, 0x20, 0x6e, 0xe9, 0x57, 0xa0, 0xd7,
            0x36, 0x0b, 0x2a, 0x0b, 0x2e, 0x92, 0xb3, 0xdf, 0x18, 0x30, 0x6a, 0x93, 0xd5, 0x9e, 0x0c, 0xa9,
            0xf1, 0xfe, 0x6a, 0x2d, 0x8b, 0x92, 0x7d, 0x39, 0x3c, 0xd0, 0x23, 0x2f, 0xd7, 0x70, 0x37, 0xf7,
            0x4a, 0xf1, 0xee, 0x3e, 0xe1, 0x2c, 0xbc, 0x97, 0xd8, 0x2d, 0xf1, 0x5b, 0xb0, 0x2e, 0xea, 0xaf,
        ])  # Unchanged
        r4 = bytes([
            0x4a, 0xf1, 0xee, 0x3e, 0xe1, 0x2c, 0xbc, 0x97, 0xd8, 0x2d, 0xf1, 0x5b, 0xb0, 0x2e, 0xea, 0xaf,
            0x0e, 0xec, 0x36, 0xd2, 0x0a, 0xf8, 0x57, 0x65, 0x2b, 0x49, 0x76, 0x1c, 0x40, 0x3e, 0x45, 0x2e,
            0x30, 0x27, 0x90, 0xf4, 0x26, 0x42, 0x4a, 0x4c, 0xaf, 0x75, 0xe8, 0x98, 0x79, 0xa7, 0x3e, 0x44,
            0x03, 0x03, 0xd9, 0x86, 0xb3, 0xef, 0x13, 0xdf, 0xcd, 0x10, 0xf1, 0xe5, 0x63, 0x6b, 0xcd, 0x2a,
            0x30, 0xe4, 0xa2, 0x16, 0xcf, 0xb7, 0xd5, 0x7c, 0x1b, 0xb0, 0x49, 0xb0, 0x4f, 0xc4, 0xd5, 0x8c,
            0x8d, 0xd8, 0x3c, 0x81, 0xe8, 0xa1, 0x73, 0xee, 0x76, 0xb8, 0xb8, 0x49, 0x4d, 0x0b, 0x37, 0x42,
            0x70, 0x4e, 0x08, 0x43, 0x2d, 0xdb, 0xf2, 0x37, 0xfe, 0x82, 0xcf, 0xc2, 0x99, 0xc4, 0xa1, 0xa9,
            0xbd, 0x8b, 0xc0, 0x0e, 0x88, 0x87, 0xcd, 0x82, 0x1c, 0x75, 0x82, 0x62, 0x16, 0x77, 0xc6, 0xfa,
            0x19, 0x5c, 0x04, 0xa7, 0x98, 0x40, 0x05, 0xbf, 0xf1, 0x98, 0x9b, 0xd8, 0x00, 0x95, 0x4a, 0x2a,
            0xea, 0x45, 0x5b, 0xb9, 0x89, 0x94, 0x9e, 0x07, 0xb8, 0xb5, 0x86, 0x6c, 0x9d, 0xb9, 0xc6, 0xef,
            0xc0, 0x45, 0x7b, 0x3b, 0xaf, 0x32, 0x53, 0x63, 0xd1, 0x49, 0x58, 0xe0, 0xda, 0x9b, 0x96, 0x86,
            0x1a, 0xfa, 0xac, 0xaf, 0x66, 0xee, 0x49, 0x0c, 0x37, 0x01, 0x60, 0x8f, 0xca, 0x80, 0x71, 0xfd,
            0x2c, 0x35, 0x59, 0xd8, 0x3d, 0xfe, 0xf8, 0xef, 0x0f, 0x3d, 0x12, 0x28, 0xef, 0x6c, 0xf3, 0xad,
            0x76, 0x33, 0x37, 0x4f, 0xae, 0x1d, 0x59, 0xe3, 0xdc, 0x14, 0x88, 0x46, 0xa6, 0x96, 0x4a, 0xb8,
            0xee, 0x6b, 0x74, 0xf4, 0x46, 0x13, 0x04, 0x45, 0xb1, 0x96, 0xc3, 0xca, 0x98, 0xcb, 0xe0, 0x9b,
            0x53, 0x5f, 0xd1, 0xf9, 0x78, 0x0a, 0x90, 0x46, 0xd7, 0xb3, 0x49, 0x2a, 0x5b, 0x89, 0x91, 0x1f,
            0x65, 0x49, 0x51, 0xf7, 0xea, 0x7b, 0x30, 0x57, 0x01, 0x72, 0xe2, 0x45, 0xe8, 0x42, 0xea, 0xd3
        ])
        r5 = bytes([
            0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
        ])
        r5out = bytes([
            0xf5, 0xbf, 0x46, 0xe4, 0x34, 0xf1, 0x73, 0x3c, 0xef, 0x6d, 0x46, 0x91, 0x3d, 0x75, 0x08, 0xcf,
            0x3f, 0x09, 0x57, 0xcd, 0x7e, 0xe5, 0x1a, 0xa9, 0x7d, 0x83, 0x99, 0xf8, 0xf6, 0xe9, 0x58, 0x39,
            0xa5, 0x37, 0xf0, 0x6c, 0x56, 0xa7, 0xef, 0x33, 0xe8, 0x11, 0x23, 0x26, 0x1b, 0x94, 0x9e, 0xc8,
            0xcc, 0x30, 0x81, 0x7b, 0x21, 0x9f, 0x6c, 0x54, 0x39, 0x20, 0x35, 0x88, 0xbb, 0xe1, 0x98, 0xf6,
            0x62, 0x6c, 0xa9, 0x28, 0x65, 0xd4, 0x70, 0xe8, 0x40, 0xb2, 0xd5, 0x3d, 0x94, 0x9d, 0x45, 0x27,
            0x0c, 0xc9, 0xf4, 0x09, 0x24, 0x12, 0x08, 0x59, 0xaf, 0x7c, 0xd2, 0xf3, 0xc4, 0x9a, 0xcd, 0x3d,
            0x6c, 0xe7, 0xe9, 0x6f, 0x22, 0x49, 0x18, 0x1e, 0x40, 0x72, 0x0b, 0x80, 0x47, 0x49, 0x52, 0x5f,
            0x33, 0x70, 0x77, 0xff, 0xbb, 0xc4, 0xf4, 0xb5, 0xdb, 0xbb, 0x3f, 0x02, 0x55, 0x73, 0xe9, 0x34,
            0xa1, 0x99, 0x04, 0xc0, 0x43, 0xef, 0x73, 0x4e, 0xc6, 0xaa, 0xc7, 0xe1, 0x07, 0x92, 0xb7, 0x18,
            0x84, 0x2f, 0x0d, 0x44, 0xa1, 0x5f, 0xe6, 0x7c, 0x15, 0x43, 0xbe, 0xa4, 0x16, 0x15, 0x03, 0xd0,
            0x12, 0x7a, 0x3a, 0xeb, 0xdc, 0xf1, 0x7e, 0x2d, 0x05, 0x5e, 0xb9, 0x21, 0xbe, 0x4e, 0xeb, 0xf9,
            0x78, 0x69, 0x40, 0x7f, 0x38, 0xe8, 0x19, 0xe2, 0x41, 0x45, 0xbf, 0x0c, 0x5f, 0xa8, 0x8c, 0xdc,
            0x1e, 0x52, 0x0e, 0x37, 0x7a, 0x39, 0xd7, 0x6a, 0x5e, 0x65, 0x06, 0x8c, 0x85, 0x5d, 0x0a, 0xc9,
            0xb4, 0xfd, 0xfe, 0x31, 0x6e, 0x8c, 0x0d, 0x7d, 0xa1, 0xad, 0x20, 0x6e, 0xe9, 0x57, 0xa0, 0xd7,
            0x36, 0x0b, 0x2a, 0x0b, 0x2e, 0x92, 0xb3, 0xdf, 0x18, 0x30, 0x6a, 0x93, 0xd5, 0x9e, 0x0c, 0xa9,
            0xf1, 0xfe, 0x6a, 0x2d, 0x8b, 0x92, 0x7d, 0x39, 0x3c, 0xd0, 0x23, 0x2f, 0xd7, 0x70, 0x37, 0xf7,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
        ])

        self.assertEqual(XeCryptBnQwNeMod(r3, r4, 0x20, 0x20), r5out)

    def test_normal_2(self):


"""Delegate test execution to another environment."""

from __future__ import absolute_import, print_function

import os
import re
import sys
import tempfile

import lib.pytar
import lib.thread

from lib.executor import (
    SUPPORTED_PYTHON_VERSIONS,
    HTTPTESTER_HOSTS,
    create_shell_command,
    run_httptester,
    start_httptester,
)

from lib.config import (
    TestConfig,
    EnvironmentConfig,
    IntegrationConfig,
    ShellConfig,
    SanityConfig,
    UnitsConfig,
)

from lib.core_ci import (
    AnsibleCoreCI,
)

from lib.manage_ci import (
    ManagePosixCI,
    ManageWindowsCI,
)

from lib.util import (
    ApplicationError,
    run_command,
    common_environment,
    pass_vars,
    display,
)

from lib.docker_util import (
    docker_exec,
    docker_get,
    docker_pull,
    docker_put,
    docker_rm,
    docker_run,
    docker_available,
    docker_network_disconnect,
    get_docker_networks,
)

from lib.cloud import (
    get_cloud_providers,
)

from lib.target import (
    IntegrationTarget,
)

def delegate(args, exclude, require, integration_targets):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    :type integration_targets: tuple[IntegrationTarget]
    :rtype: bool
    """
    if isinstance(args, TestConfig):
        with tempfile.NamedTemporaryFile(prefix='metadata-', suffix='.json', dir=os.getcwd()) as metadata_fd:
            args.metadata_path = os.path.basename(metadata_fd.name)
            args.metadata.to_file(args.metadata_path)

            try:
                return delegate_command(args, exclude, require, integration_targets)
            finally:
                args.metadata_path = None
    else:
        return delegate_command(args, exclude, require, integration_targets)

def delegate_command(args, exclude, require, integration_targets):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    :type integration_targets: tuple[IntegrationTarget]
    :rtype: bool
    """
    if args.tox:
        delegate_tox(args, exclude, require, integration_targets)
        return True

    if args.docker:
        delegate_docker(args, exclude, require, integration_targets)
        return True

    if args.remote:
        delegate_remote(args, exclude, require, integration_targets)
        return True

    return False

def delegate_tox(args, exclude, require, integration_targets):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    :type integration_targets: tuple[IntegrationTarget]
    """
    if args.python:
        versions = (args.python_version,)

        if args.python_version not in SUPPORTED_PYTHON_VERSIONS:
            raise ApplicationError('tox does not support Python version %s' % args.python_version)
    else:
        versions = SUPPORTED_PYTHON_VERSIONS

    if args.httptester:
        needs_httptester = sorted(target.name for target in integration_targets if 'needs/httptester/' in target.aliases)

        if needs_httptester:
            display.warning('Use --docker or --remote to enable httptester for tests marked "needs/httptester": %s' % ', '.join(needs_httptester))

    options = {
        '--tox': args.tox_args,
        '--tox-sitepackages': 0,
    }

    for version in versions:
        tox = ['tox', '-c', 'test/runner/tox.ini', '-e', 'py' + version.replace('.', '')]

        if args.tox_sitepackages:
            tox.append('--sitepackages')

        tox.append('--')

        cmd = generate_command(args, os.path.abspath('bin/ansible-test'), options, exclude, require)

        if not args.python:
            cmd += ['--python', version]

        if isinstance(args, TestConfig):
            if args.coverage and not args.coverage_label:
                cmd += ['--coverage-label', 'tox-%s' % version]

        env = common_environment()

        # temporary solution to permit ansible-test delegated to tox to provision remote resources
        optional = (
            'SHIPPABLE',
            'SHIPPABLE_BUILD_ID',
            'SHIPPABLE_JOB_NUMBER',
        )

        env.update(pass_vars(required=[], optional=optional))

        run_command(args, tox + cmd, env=env)

def delegate_docker(args, exclude, require, integration_targets):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    :type integration_targets: tuple[IntegrationTarget]
    """
    test_image = args.docker
    privileged = args.docker_privileged

    if isinstance(args, ShellConfig):
        use_httptester = args.httptester
    else:
        use_httptester = args.httptester and any('needs/httptester/' in target.aliases for target in integration_targets)

    if use_httptester:
        docker_pull(args, args.httptester)

    docker_pull(args, test_image)

    httptester_id = None
    test_id = None

    options = {
        '--docker': 1,
        '--docker-privileged': 0,
        '--docker-util': 1,
    }

    cmd = generate_command(args, '/root/ansible/bin/ansible-test', options, exclude, require)

    if isinstance(args, TestConfig):
        if args.coverage and not args.coverage_label:
            image_label = args.docker_raw
            image_label = re.sub('[^a-zA-Z0-9]+', '-', image_label)
            cmd += ['--coverage-label', 'docker-%s' % image_label]

    if isinstance(args, IntegrationConfig):
        if not args.allow_destructive:
            cmd.append('--allow-destructive')

    cmd_options = []

    if isinstance(args, ShellConfig) or (isinstance(args, IntegrationConfig) and args.debug_strategy):
        cmd_options.append('-it')

    with tempfile.NamedTemporaryFile(prefix='ansible-source-', suffix='.tgz') as local_source_fd:
        try:
            if not args.explain:
                if args.docker_keep_git:
                    tar_filter = lib.pytar.AllowGitTarFilter()
                else:
                    tar_filter = lib.pytar.DefaultTarFilter()

                lib.pytar.create_tarfile(local_source_fd.name, '.', tar_filter)

            if use_httptester:
                httptester_id = run_httptester(args)
            else:
                httptester_id = None

            test_options = [
                '--detach',
                '--volume', '/sys/fs/cgroup:/sys/fs/cgroup:ro',
                '--privileged=%s' % str(privileged).lower(),
            ]

            if args.docker_memory:
                test_options.extend([
                    '--memory=%d' % args.docker_memory,
                    '--memory-swap=%d' % args.docker_memory,
                ])

            docker_socket = '/var/run/docker.sock'

            if args.docker_seccomp != 'default':
                test_options += ['--security-opt', 'seccomp=%s' % args.docker_seccomp]

            if os.path.exists(docker_socket):
                test_options += ['--volume', '%s:%s' % (docker_socket, docker_socket)]

            if httptester_id:
                test_options += ['--env', 'HTTPTESTER=1']

                for host in HTTPTESTER_HOSTS:
                    test_options += ['--link', '%s:%s' % (httptester_id, host)]

            if isinstance(args, IntegrationConfig):
                cloud_platforms = get_cloud_providers(args)

                for cloud_platform in cloud_platforms:
                    test_options += cloud_platform.get_docker_run_options()

            test_id, _ = docker_run(args, test_image, options=test_options)

            if args.explain:
                test_id = 'test_id'
            else:
                test_id = test_id.strip()

            # write temporary files to /root since /tmp isn't ready immediately on container start
            docker_put(args, test_id, 'test/runner/setup/docker.sh', '/root/docker.sh')
            docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh'])
            docker_put(args, test_id, local_source_fd.name, '/root/ansible.tgz')
            docker_exec(args, test_id, ['mkdir', '/root/ansible'])
            docker_exec(args, test_id, ['tar', 'oxzf', '/root/ansible.tgz', '-C', '/root/ansible'])

            # docker images are only expected to have a single python version available
            if isinstance(args, UnitsConfig) and not args.python:
                cmd += ['--python', 'default']

            # run unit tests unprivileged to prevent stray writes to the source tree
            # also disconnect from the network once requirements have been installed
            if isinstance(args, UnitsConfig):
                writable_dirs = [
                    '/root/ansible/.pytest_cache',
                ]

                docker_exec(args, test_id, ['mkdir', '-p'] + writable_dirs)
                docker_exec(args, test_id, ['chmod', '777'] + writable_dirs)

                docker_exec(args, test_id, ['find', '/root/ansible/test/results/', '-type', 'd', '-exec', 'chmod', '777', '{}', '+'])

                docker_exec(args, test_id, ['chmod', '755', '/root'])
                docker_exec(args, test_id, ['chmod', '644', '/root/ansible/%s' % args.metadata_path])

                docker_exec(args, test_id, ['useradd', 'pytest', '--create-home'])

                docker_exec(args, test_id, cmd + ['--requirements-mode', 'only'], options=cmd_options)

                networks = get_docker_networks(args, test_id)

                for network in networks:
                    docker_network_disconnect(args, test_id, network)

                cmd += ['--requirements-mode', 'skip']