<commit_before>"""
Docker is a server-client architecture where the client interacts with the server
via http. This means the server doesn't see the directory where the client is
executing from and thus has no access to files around the Dockerfile.

This means that to include anything from the local directory, it must first be
sent to the server in a zip file.

Here we define the class that builds this "context" zip file.
"""

from harpoon.helpers import a_temp_file
from harpoon.errors import HarpoonError

from delfick_app import command_output
from contextlib import contextmanager
from gitmit.mit import GitTimes
import tempfile
import fnmatch
import logging
import tarfile
import shutil
import json
import six
import os
import re

log = logging.getLogger("harpoon.ship.context")

class ContextWrapper(object):
    """Wraps a tarfile context, so we can continue changing it afterwards"""
    def __init__(self, t, tmpfile):
        self.t = t
        self.tmpfile = tmpfile

    def close(self):
        self.t.close()
        self.tmpfile.flush()
        self.tmpfile.seek(0)

    @property
    def name(self):
        return self.tmpfile.name

    @contextmanager
    def clone_with_new_dockerfile(self, conf, docker_file):
        """Clone this tarfile and add in another filename before closing the new tar and returning"""
        log.info("Copying context to add a different dockerfile")
        self.close()
        with a_temp_file() as tmpfile:
            old_t = os.stat(self.tmpfile.name).st_size > 0
            if old_t:
                shutil.copy(self.tmpfile.name, tmpfile.name)

            with tarfile.open(tmpfile.name, mode="a") as t:
                conf.add_docker_file_to_tarfile(docker_file, t)
                yield ContextWrapper(t, tmpfile)

class ContextBuilder(object):
    """
    Understands how to build a context

    Can take into account git to determine what to include and exclude.
    """
    @contextmanager
    def make_context(self, context, silent_build=False, extra_context=None):
        """
        Context manager for creating the context of the image

        Arguments:

        context - ``harpoon.option_spec.image_objs.Context``
            Knows all the context related options

        docker_file - ``harpoon.option_spec.image_objs.Dockerfile``
            Knows what is in the dockerfile and it's mtime

        silent_build - boolean
            If True, then suppress printing out information

        extra_context - List of (string, string)
            First string represents the content to put in a file and the second
            string represents where in the context this extra file should go
        """
        with a_temp_file() as tmpfile:
            t = tarfile.open(mode='w', fileobj=tmpfile)
            for thing, mtime, arcname in self.find_mtimes(context, silent_build):
                if mtime:
                    os.utime(thing, (mtime, mtime))

                log.debug("Context: {0}".format(arcname))
                t.add(thing, arcname=arcname)

            if extra_context:
                extra = list(extra_context)
                for content, arcname in extra:
                    mtime_match = re.search("mtime\((\d+)\)$", arcname)
                    specified_mtime = None if not mtime_match else int(mtime_match.groups()[0])

                    with self.the_context(content, silent_build=silent_build) as fle:
                        if specified_mtime:
                            os.utime(fle.name, (specified_mtime, specified_mtime))

                        log.debug("Context: {0}".format(arcname))
                        t.add(fle.name, arcname=arcname)

            yield ContextWrapper(t, tmpfile)

    @contextmanager
    def the_context(self, content, silent_build=False):
        """Return either a file with the content written to it, or a whole new context tar"""
        if isinstance(content, six.string_types):
            with a_temp_file() as fle:
                fle.write(content.encode('utf-8'))
                fle.seek(0)
                yield fle
        else:
            with ContextBuilder().make_context(content["context"], silent_build=silent_build) as wrapper:
                wrapper.close()
                yield wrapper.tmpfile

    def find_mtimes(self, context, silent_build):
        """
        Return [(filename, mtime), ...] for all the files.

        Where the mtime comes from git or is None depending on the value of
        use_git_timestamps

        use_git_timestamps can be True indicating all files or it can be a list
        of globs indicating which files should receive git timestamps.
        """
        if not context.enabled:
            return
<commit_msg>Making context aware of pulling paths from images<commit_after>"""
Docker is a server-client architecture where the client interacts with the server
via http. This means the server doesn't see the directory where the client is
executing from and thus has no access to files around the Dockerfile.

This means that to include anything from the local directory, it must first be
sent to the server in a zip file.

Here we define the class that builds this "context" zip file.
"""

from harpoon.helpers import a_temp_file
from harpoon.errors import HarpoonError

from delfick_app import command_output
from contextlib import contextmanager
from gitmit.mit import GitTimes
from six.moves import StringIO
import tempfile
import tarfile
import fnmatch
import logging
import tarfile
import shutil
import json
import six
import os
import re

log = logging.getLogger("harpoon.ship.context")

class ContextWrapper(object):
    """Wraps a tarfile context, so we can continue changing it afterwards"""
    def __init__(self, t, tmpfile):
        self.t = t
        self.tmpfile = tmpfile

    def close(self):
        self.t.close()
        self.tmpfile.flush()
        self.tmpfile.seek(0)

    @property
    def name(self):
        return self.tmpfile.name

    @contextmanager
    def clone_with_new_dockerfile(self, conf, docker_file):
        """Clone this tarfile and add in another filename before closing the new tar and returning"""
        log.info("Copying context to add a different dockerfile")
        self.close()
        with a_temp_file() as tmpfile:
            old_t = os.stat(self.tmpfile.name).st_size > 0
            if old_t:
                shutil.copy(self.tmpfile.name, tmpfile.name)

            with tarfile.open(tmpfile.name, mode="a") as t:
                conf.add_docker_file_to_tarfile(docker_file, t)
                yield ContextWrapper(t, tmpfile)

class ContextBuilder(object):
    """
    Understands how to build a context

    Can take into account git to determine what to include and exclude.
    """
    @contextmanager
    def make_context(self, context, silent_build=False, extra_context=None):
        """
        Context manager for creating the context of the image

        Arguments:

        context - ``harpoon.option_spec.image_objs.Context``
            Knows all the context related options

        docker_file - ``harpoon.option_spec.image_objs.Dockerfile``
            Knows what is in the dockerfile and it's mtime

        silent_build - boolean
            If True, then suppress printing out information

        extra_context - List of (content, string)
            content is either a string repsenting the content to put in a file
            or a dictionary representing what path to get from what docker image

            The second string represents where in the context this extra file should go
        """
        with a_temp_file() as tmpfile:
            t = tarfile.open(mode='w', fileobj=tmpfile)
            for thing, mtime, arcname in self.find_mtimes(context, silent_build):
                if mtime:
                    os.utime(thing, (mtime, mtime))

                log.debug("Context: {0}".format(arcname))
                t.add(thing, arcname=arcname)

            if extra_context:
                extra = list(extra_context)
                for content, arcname in extra:
                    mtime_match = re.search("mtime\((\d+)\)$", arcname)
                    specified_mtime = None if not mtime_match else int(mtime_match.groups()[0])

                    with self.the_context(content, silent_build=silent_build) as fle:
                        if specified_mtime:
                            os.utime(fle.name, (specified_mtime, specified_mtime))

                        log.debug("Context: {0}".format(arcname))
                        t.add(fle.name, arcname=arcname)

            yield ContextWrapper(t, tmpfile)

    @contextmanager
    def the_context(self, content, silent_build=False):
        """Return either a file with the content written to it, or a whole new context tar"""
        if isinstance(content, six.string_types):
            with a_temp_file() as fle:
                fle.write(content.encode('utf-8'))
                fle.seek(0)
                yield fle
        elif "context" in content:
            with ContextBuilder().make_context(content["context"], silent_build=silent_build) as wrapper:
                wrapper.close()
                yield wrapper.tmpfile
        elif "image" in content:
            from harpoon.ship.builder import Builder
            from harpoon.ship.runner import Runner
            with a_temp_file() as fle:
                content["conf"].command = "yes"
                with Runner()._run_container(content["conf"], content["images"], detach=True, delete_anyway=True):
                    try:
                        strm, stat = content["docker_context"].get_archive(content["conf"].container_id, content["path"])
                    except docker.errors.NotFound:
                        raise BadOption("Trying to get something from an image that don't exist!", path=content["path"], image=content["conf"].image_name)
                    else:
                        log.debug(stat)

                        fo = StringIO(strm.read())
                        tf = tarfile.TarFile(fileobj=fo)

                        if tf.firstmember.isdir():
                            tf2 = tarfile.TarFile(fileobj=fle, mode='w')
                            name = tf.firstmember.name
                            for member in tf.getmembers()[1:]:
                                member.name = member.name[len(name)+1:]
                                if not member.isdir():
                                    tf2.addfile(member, fileobj=tf.extractfile(member.name))
                            tf2.close()
                        else:
                            fle.write(tf.extractfile(tf.firstmember.name).read())

                        tf.close()
                        log.info("Got '{0}' from {1} for context".format(content["path"], content["conf"].container_id))

                fle.seek(0)
                yield fle

    def find_mtimes(self, context, silent_build):
        """
        Return [(filename, mtime), ...] for all the files.

        Where the mtime comes from git or is None depending on the value of
        use_git_timestamps

        use_git_timestamps can be True indicating all files or it can be a list
        of globs indicating which files should receive git timestamps.
        """
        if not context.enabled:
            return

{'fraction_non_alphanumeric': 0.07942110836569008, 'fraction_numerical': 0.001941404871161313, 'mean_word_length': 2.09306768558952, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6238496', 'n_tokens_mistral': 3055, 'n_tokens_neox': 2919, 'n_words': 985}