<commit_before>import shutil
import os
import types

import util

def create_source(src, **keywds):
    """
    Wrapper to create a Source object.

    @param src          - Path to the source object.
    @param **keywds     - Passed to the underlying source object.
    """
    ret = None
    if type(src) == types.StringType:
        if src.startswith("git://"):
            ret = GitSource(src, **keywds)
        elif src.startswith("file://"):
            ret = FileSource(src, **keywds)
    elif type(src) == types.FunctionType:
        ret  = FuncSource(src, **keywds)

    if not ret:
        raise util.InhibitorError("Unknown source type: %s" % (src,))
    return ret

class _GenericSource(object):
    """
    Generic source object.

    @param src                  - Source path.
    @param inhibitor_state      - Inhibitor Configuration
    @param dest                 - Destination within the stage4 chroot.
    @param keep                 - Leave output in the stage4 build.
    @param mountable            - Will bind mounting suffice to provide this source.
    @param cachedir             - Name of the cache directory.
    @param ignore               - Files to be ignored when installing.  Either a function
                                  returning true or false for a given path or a list of
                                  patterns to be ignored.  See shutil.ignore_patterns.
    """
    def __init__(self, src,
            inhibitor_state = None,
            dest            = None,
            keep            = False,
            mountable       = False,
            cachedir        = None,
            ignore          = None,
            **keywds                ):
        self.src        = src
        self.istate     = inhibitor_state
        self.dest       = dest
        self.keep       = keep
        self.mountable  = mountable
        self.cachedir   = cachedir
        self.mount      = None
        self.installed  = []

        if ignore:
            self.ignore = ignore
        else:
            self.ignore = shutil.ignore_patterns('.svn', '.git', '*.swp')

    def post_conf(self, inhibitor_state):
        self.istate = inhibitor_state
        if self.cachedir and not self.cachedir.startswith(self.istate.paths.cache):
            self.cachedir = self.istate.paths.cache.pjoin(self.cachedir)

    def init(self):
        raise util.InhibitorError("init() is undefined for %s" % (self.src,))

    def file_copy_callback(self, _, targ):
        self.installed.append(targ)

    def remove(self):
        if self.mount:
            util.umount(self.mount, self.istate.mount_points)
        elif not self.keep:
            for f in self.installed:
                d = os.path.dirname(f)
                os.unlink(f)
                if len( os.listdir(d) ) == 0:
                    os.rmdir(d)

    def finish(self):
        if not self.cachedir:
            return
        shutil.rmtree(self.cachedir)

    def install(self, root):
        src = self.src
        if self.cachedir:
            src = self.cachedir
        full_dest = root.pjoin(self.dest)

        if not self.keep and self.mountable:
            util.dbg("Bind mounting %s at %s" % (src, full_dest))
            self.mount = util.Mount(src, self.dest, root)
            util.mount(self.mount, self.istate.mount_points)
        else:
            util.path_sync(
                src,
                full_dest,
                root = root,
                ignore = self.ignore,
                file_copy_callback = self.file_copy_callback
            )
        return

class FileSource(_GenericSource):
    """
    File or directory backed source object.

    @param src                  - Source path.
    @param inhibitor_state      - Inhibitor Configuration
    @param dest                 - Destination within the stage4 chroot.
    @param keep                 - Leave output in the stage4 build.

    Inherited from _GenericSource:
    @param mountable            - Will bind mounting suffice to provide this source.
    @param cachedir             - Name of the cache directory.
    @param ignore               - Files to be ignored when installing.  Either a function
                                  returning true or false for a given path or a list of
                                  patterns to be ignored.  See shutil.ignore_patterns.
    """
    def __init__(self, src, inhibitor_state = None, dest = None, keep = False, **keywds):
        real_src    = util.Path(src[6:])
        mountable   = False

        if not os.path.lexists(real_src):
            raise util.InhibitorError("Path %s does not exist" % real_src)
        elif os.path.isdir(real_src):
            mountable = True

        super(FileSource, self).__init__(
            src             = real_src,
            inhibitor_state = inhibitor_state,
            dest            = dest,
            keep            = keep,
            mountable       = mountable,
            **keywds
        )

    def init(self):
        # No need to fetch, we either bind mount or copy depending on if this
        # will be left in the chroot and is a directory or not.
        return


class FuncSource(_GenericSource):
    """
    Function backed source object.  If the function returns a string, it will be
    written to dest.  If the function returns a (nested) dictionary, then starting
    at dest, the path is represented by dictionary keys and the contents of the
    files by the values (strings).

    @param src                  - Source path.<commit_msg>source:  allow mountable to be set on FileSources<commit_after>import shutil
import os
import types

import util

def create_source(src, **keywds):
    """
    Wrapper to create a Source object.

    @param src          - Path to the source object.
    @param **keywds     - Passed to the underlying source object.
    """
    ret = None
    if type(src) == types.StringType:
        if src.startswith("git://"):
            ret = GitSource(src, **keywds)
        elif src.startswith("file://"):
            ret = FileSource(src, **keywds)
    elif type(src) == types.FunctionType:
        ret  = FuncSource(src, **keywds)

    if not ret:
        raise util.InhibitorError("Unknown source type: %s" % (src,))
    return ret

class _GenericSource(object):
    """
    Generic source object.

    @param src                  - Source path.
    @param inhibitor_state      - Inhibitor Configuration
    @param dest                 - Destination within the stage4 chroot.
    @param keep                 - Leave output in the stage4 build.
    @param mountable            - Will bind mounting suffice to provide this source.
    @param cachedir             - Name of the cache directory.
    @param ignore               - Files to be ignored when installing.  Either a function
                                  returning true or false for a given path or a list of
                                  patterns to be ignored.  See shutil.ignore_patterns.
    """
    def __init__(self, src,
            inhibitor_state = None,
            dest            = None,
            keep            = False,
            mountable       = False,
            cachedir        = None,
            ignore          = None,
            **keywds                ):
        self.src        = src
        self.istate     = inhibitor_state
        self.dest       = dest
        self.keep       = keep
        self.mountable  = mountable
        self.cachedir   = cachedir
        self.mount      = None
        self.installed  = []

        if ignore:
            self.ignore = ignore
        else:
            self.ignore = shutil.ignore_patterns('.svn', '.git', '*.swp')

    def post_conf(self, inhibitor_state):
        self.istate = inhibitor_state
        if self.cachedir and not self.cachedir.startswith(self.istate.paths.cache):
            self.cachedir = self.istate.paths.cache.pjoin(self.cachedir)

    def init(self):
        raise util.InhibitorError("init() is undefined for %s" % (self.src,))

    def file_copy_callback(self, _, targ):
        self.installed.append(targ)

    def remove(self):
        if self.mount:
            util.umount(self.mount, self.istate.mount_points)
        elif not self.keep:
            for f in self.installed:
                d = os.path.dirname(f)
                os.unlink(f)
                if len( os.listdir(d) ) == 0:
                    os.rmdir(d)

    def finish(self):
        if not self.cachedir:
            return
        shutil.rmtree(self.cachedir)

    def install(self, root):
        src = self.src
        if self.cachedir:
            src = self.cachedir
        full_dest = root.pjoin(self.dest)

        if not self.keep and self.mountable:
            util.dbg("Bind mounting %s at %s" % (src, full_dest))
            self.mount = util.Mount(src, self.dest, root)
            util.mount(self.mount, self.istate.mount_points)
        else:
            util.path_sync(
                src,
                full_dest,
                root = root,
                ignore = self.ignore,
                file_copy_callback = self.file_copy_callback
            )
        return

class FileSource(_GenericSource):
    """
    File or directory backed source object.

    @param src                  - Source path.
    @param inhibitor_state      - Inhibitor Configuration
    @param dest                 - Destination within the stage4 chroot.
    @param keep                 - Leave output in the stage4 build.

    Inherited from _GenericSource:
    @param mountable            - Will bind mounting suffice to provide this source.
    @param cachedir             - Name of the cache directory.
    @param ignore               - Files to be ignored when installing.  Either a function
                                  returning true or false for a given path or a list of
                                  patterns to be ignored.  See shutil.ignore_patterns.
    """
    def __init__(self, src, inhibitor_state = None, dest = None, keep = False, mountable=None, **keywds):
        real_src    = util.Path(src[6:])

        if not os.path.lexists(real_src):
            raise util.InhibitorError("Path %s does not exist" % real_src)

        if mountable == None:
            mountable = True if os.path.isdir(real_src) else False

        super(FileSource, self).__init__(
            src             = real_src,
            inhibitor_state = inhibitor_state,
            dest            = dest,
            keep            = keep,
            mountable       = mountable,
            **keywds
        )

    def init(self):
        # No need to fetch, we either bind mount or copy depending on if this
        # will be left in the chroot and is a directory or not.
        return


class FuncSource(_GenericSource):
    """
    Function backed source object.  If the function returns a string, it will be
    written to dest.  If the function returns a (nested) dictionary, then starting
    at dest, the path is represented by dictionary keys and the contents of the
    files by the values (strings).

    @param src                  - Source path.
{'fraction_non_alphanumeric': 0.07985643786451324, 'fraction_numerical': 0.0010767160161507402, 'mean_word_length': 1.5717581910475311, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1832451', 'n_tokens_mistral': 3038, 'n_tokens_neox': 2936, 'n_words': 952}