<commit_before>#
# Copyright (c) 2004-2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
"""
Base classes and data used for all policy
"""

import imp
import os
import sys
import types

from conary.lib import util, log, graph
from conary.build import action, errors, filter


# buckets (enum -- but may possibly work someday as bitmask for policy
# that could run more than once in different contexts)
TESTSUITE            = 1 << 0
DESTDIR_PREPARATION  = 1 << 1
DESTDIR_MODIFICATION = 1 << 2
PACKAGE_CREATION     = 1 << 3
PACKAGE_MODIFICATION = 1 << 4
ENFORCEMENT          = 1 << 5
ERROR_REPORTING      = 1 << 6

# requirements (sparse bitmask, 5 sensible states)
REQUIRED               = 1 << 0
ORDERED                = 1 << 1
PRIOR                  = 1 << 2
REQUIRED_PRIOR         = REQUIRED|ORDERED|PRIOR
REQUIRED_SUBSEQUENT    = REQUIRED|ORDERED
CONDITIONAL_PRIOR      = ORDERED|PRIOR
CONDITIONAL_SUBSEQUENT = ORDERED

# file trees
NO_FILES = 0 << 0
DESTDIR  = 1 << 0
BUILDDIR = 1 << 1
DIR      = DESTDIR|BUILDDIR
PACKAGE  = 1 << 2


class Policy(action.RecipeAction):
    """
    Abstract superclass for all policy actions.  Policy actions
    that operate on the entire C{%(destdir)s} implement the C{do} method;
    Policy actions that operate on a per-file basis implement the
    C{doFile} method.  The C{doFile} function is never called for files
    that match an exception regexp; C{do} functions must implement
    their own exception regexp handling.

    The class variables below apply automatically to the C{doFile}
    method; classes implementing the C{do} method instead may use them as
    well, but should use the same rules if they do use them.  All of
    them have C{self.macros} applied before use.

    @cvar invariantsubtrees: if C{invariantsubtrees} is not empty,
    then it is a list of subtrees (relative to C{%(destdir)s}) to
    walk INSTEAD of walking the entire C{%(destdir)s} tree.  Any
    C{subtrees} are appended to C{invariantsubtrees}.

    @cvar invariantinclusions: if C{invariantinclusions} is not empty,
    then only files matching a filter in it are considered to be passed
    to to the C{doFile} method.  Any exceptions, including invariants,
    are applied after C{invariantinclusions} are applied; this means
    that all exceptions OVERRULE every type of inclusion.

    @cvar invariantexceptions: subclasses may set to a list of
    exception filters that are always applied regardless of what other
    exceptions may be provided by the recipe; these exceptions being
    applied is an invariant condition of the C{doFile} method.

    @cvar recursive: if True, walk entire subtrees; if False,
    work only on contents of listed directories (C{invariantsubtrees}
    and C{subtrees}).
    @type recursive: boolean

    @cvar filetree: where to look for files to which to apply the
    policy: C{policy.DESTDIR}, the default, walks the files in the
    C{destdir}, C{policy.BUILDDIR} walks the files in the build
    directory, and C{policy.PACKAGE} iterates over the packaged
    files rather than walking the destdir, which besides being
    possibly faster also applies to files that are not on the
    filesystem (like device nodes).

    @cvar rootdir: The root of the tree to walk for files, normally
    implied by the setting of filetree.

    @cvar processUnmodified: allows special handling for derived
    packages in order to make only appropriate changes in the
    derived package.  C{None} (default) disables the policy for
    derived packages.  C{True} causes C{doFile} to be called for
    all files found, regardless of whether they are in the
    parent version, and C{False} causes C{doFile} to be called
    only for files that are new or have changed timestamps.
    Note that if C{filetree} is C{policy.PACKAGE}, unchanged
    file contents will lead to unchanged timestamps.
    """
    bucket = None
    invariantsubtrees = []
    invariantexceptions = []
    invariantinclusions = []
    recursive = True
    filetree = DESTDIR
    rootdir = None
    processUnmodified = None

    keywords = {
        'use': None,
        'exceptions': None,
        'inclusions': None,
	'subtrees': None,
    }

    def __init__(self, recipe, **keywords):
	"""
	@keyword exceptions: Optional argument; regexp(s) specifying
	files to ignore while taking the policy action.  It will be
	interpolated against recipe macros before being used.
	@type exceptions: None, filter string/tuple, or
	tuple/list of filter strings/tuples
	@keyword use: Optional argument; Use flag(s) telling whether
	to actually perform the action.
	@type use: None, Use flag, or tuple/list of Use flags
        @keyword subtree: Subtree to which to limit the policy, or it
        it already is limited (invariantsubtrees), then additional
        subtrees to consider.
        @type subtree: string or sequence of strings
        @keyword inclusions: C{FileFilter}s to which to limit the policy,
        or if it already is limited (invariantinclusion) then additional
        C{FileFilter}s to include within the general limitation.
        @type inclusions: C{FileFilter} strings, C{FileFilter} tuples,
        or list (not tuple) of C{FileFilter} strings or C{FileFilter} tuples.
	"""
	# enforce abstract base class status
	assert(self.__class__ is not Policy)

	action.RecipeAction.__init__(self, None, [], **keywords)
	self.recipe = recipe
        if self.rootdir is None and self.filetree:
            self.rootdir = {
                DESTDIR: '%(destdir)s',
                BUILDDIR: '%(builddir)s',
                PACKAGE: '',
            }[self.filetree]


    def postInit(self):
        """
        Hook for initialization that cannot happen until after all
        policies have been loaded into the recipe and thus cannot
        happen in __init__; mainly for policies that need to pass
        information to other policies at initialization time.
        """
        pass

    def updateArgs(self, *args, **keywords):
	"""
	The default way to update a class is to override any provided
	keywords.  Subclasses which have the ability to provide more
	intelligent handling can override this method.  This method
	is invoked automatically by recipe.py when a recipe references
	a policy object.  It acts rather like __init__ except that it
	can meaningfully be called more than once for an object.

	Some keyword arguments (at least C{exceptions} and C{subtrees})
	should be appended rather than replaced.
	"""
	exceptions = keywords.pop('exceptions', None)
	if exceptions:
	    if not self.exceptions:
		self.exceptions = []
            if type(exceptions) in (list, tuple):
                self.exceptions.extend(exceptions)
            else:
	        self.exceptions.append(exceptions)
	subtrees = keywords.pop('subtrees', None)
	if subtrees:
	    if not self.subtrees:
		self.subtrees = []
            if type(subtrees) in (list, tuple):
	        self.subtrees.extend(subtrees)
            else:
	        self.subtrees.append(subtrees)

	inclusions = keywords.pop('inclusions', [])
	if (args or inclusions) and not self.inclusions:
            self.inclusions = []

        if inclusions:
            if type(inclusions) == list:
                self.inclusions.extend(inclusions)
            else:
                self.inclusions.append(inclusions)

        if args:
	    self.inclusions.extend(args)

	self.addArgs(**keywords)

    def filterExpression(self, expression, name=None):
        """
        backwards compatibility
        """
        args, kwargs = self.filterExpArgs(expression, name)
        return (args[0],
                args[1],
                kwargs['setmode'],
                kwargs['unsetmode'],
                kwargs['name'],
                kwargs['rootdir'])

    def filterExpArgs(self, expression, name=None):
	"""
	@param expression: regular expression or tuple of
	(regex, [setmode, [unsetmode]])
        Creates arguments to filter.Filter.__init__
        """
        kwargs = {
            'name': name,
            'setmode': None,
            'unsetmode': None,
            'rootdir': self.rootdir,
        }
        macros = self.macros
	if type(expression) in (str, types.FunctionType):
            return (expression, macros), kwargs

	if type(expression) is not list:
	    expression = list(expression)

        # this normally happens when code at a higher level
        # has a filterExp tuple in a list of items and does
        # not need to know how to handle it separately.
        if len(expression) == 1 and type(expression[0]) in (list, tuple):
            expression = list(expression[0])

        regex = expression.pop(0)
        if expression:
            kwargs['setmode'] = expression.pop(0)
        if expression:
            kwargs['unsetmode'] = expression.pop(0)
        return (regex, macros), kwargs

    def compileFilters(self, expressionList, filterList):
        seen = []
	for expression in expressionList:
            if expression in seen:
                # only put each expression on the list once
                continue
            seen.append(expression)
            args, kwargs = self.filterExpArgs(expression)
            filterList.append(filter.Filter(*args, **kwargs))

    def doProcess(self, recipe):
	"""
	Invocation instance
        @param recipe: holds the recipe object, which is used for
	the macro set and package objects.
        @return: None
        @rtype: None
	"""
	self.recipe = recipe
	self.macros = recipe.macros

        if self.rootdir:
            self.rootdir = self.rootdir % self.macros

        if (hasattr(recipe, '_isDerived')
            and recipe._isDerived == True
            and self.processUnmodified is None):
            # This policy does not handle derived packages
            return

	if hasattr(self.__class__, 'preProcess'):
	    self.preProcess()<commit_msg>use glob and regexp objects interchangeably (CNY-2222)<commit_after>#
# Copyright (c) 2004-2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
"""
Base classes and data used for all policy
"""

import imp
import os
import sys
import types

from conary.lib import util, log, graph
from conary.build import action, errors, filter


# buckets (enum -- but may possibly work someday as bitmask for policy
# that could run more than once in different contexts)
TESTSUITE            = 1 << 0
DESTDIR_PREPARATION  = 1 << 1
DESTDIR_MODIFICATION = 1 << 2
PACKAGE_CREATION     = 1 << 3
PACKAGE_MODIFICATION = 1 << 4
ENFORCEMENT          = 1 << 5
ERROR_REPORTING      = 1 << 6

# requirements (sparse bitmask, 5 sensible states)
REQUIRED               = 1 << 0
ORDERED                = 1 << 1
PRIOR                  = 1 << 2
REQUIRED_PRIOR         = REQUIRED|ORDERED|PRIOR
REQUIRED_SUBSEQUENT    = REQUIRED|ORDERED
CONDITIONAL_PRIOR      = ORDERED|PRIOR
CONDITIONAL_SUBSEQUENT = ORDERED

# file trees
NO_FILES = 0 << 0
DESTDIR  = 1 << 0
BUILDDIR = 1 << 1
DIR      = DESTDIR|BUILDDIR
PACKAGE  = 1 << 2


class Policy(action.RecipeAction):
    """
    Abstract superclass for all policy actions.  Policy actions
    that operate on the entire C{%(destdir)s} implement the C{do} method;
    Policy actions that operate on a per-file basis implement the
    C{doFile} method.  The C{doFile} function is never called for files
    that match an exception regexp; C{do} functions must implement
    their own exception regexp handling.

    The class variables below apply automatically to the C{doFile}
    method; classes implementing the C{do} method instead may use them as
    well, but should use the same rules if they do use them.  All of
    them have C{self.macros} applied before use.

    @cvar invariantsubtrees: if C{invariantsubtrees} is not empty,
    then it is a list of subtrees (relative to C{%(destdir)s}) to
    walk INSTEAD of walking the entire C{%(destdir)s} tree.  Any
    C{subtrees} are appended to C{invariantsubtrees}.

    @cvar invariantinclusions: if C{invariantinclusions} is not empty,
    then only files matching a filter in it are considered to be passed
    to to the C{doFile} method.  Any exceptions, including invariants,
    are applied after C{invariantinclusions} are applied; this means
    that all exceptions OVERRULE every type of inclusion.

    @cvar invariantexceptions: subclasses may set to a list of
    exception filters that are always applied regardless of what other
    exceptions may be provided by the recipe; these exceptions being
    applied is an invariant condition of the C{doFile} method.

    @cvar recursive: if True, walk entire subtrees; if False,
    work only on contents of listed directories (C{invariantsubtrees}
    and C{subtrees}).
    @type recursive: boolean

    @cvar filetree: where to look for files to which to apply the
    policy: C{policy.DESTDIR}, the default, walks the files in the
    C{destdir}, C{policy.BUILDDIR} walks the files in the build
    directory, and C{policy.PACKAGE} iterates over the packaged
    files rather than walking the destdir, which besides being
    possibly faster also applies to files that are not on the
    filesystem (like device nodes).

    @cvar rootdir: The root of the tree to walk for files, normally
    implied by the setting of filetree.

    @cvar processUnmodified: allows special handling for derived
    packages in order to make only appropriate changes in the
    derived package.  C{None} (default) disables the policy for
    derived packages.  C{True} causes C{doFile} to be called for
    all files found, regardless of whether they are in the
    parent version, and C{False} causes C{doFile} to be called
    only for files that are new or have changed timestamps.
    Note that if C{filetree} is C{policy.PACKAGE}, unchanged
    file contents will lead to unchanged timestamps.
    """
    bucket = None
    invariantsubtrees = []
    invariantexceptions = []
    invariantinclusions = []
    recursive = True
    filetree = DESTDIR
    rootdir = None
    processUnmodified = None

    keywords = {
        'use': None,
        'exceptions': None,
        'inclusions': None,
	'subtrees': None,
    }

    def __init__(self, recipe, **keywords):
	"""
	@keyword exceptions: Optional argument; regexp(s) specifying
	files to ignore while taking the policy action.  It will be
	interpolated against recipe macros before being used.
	@type exceptions: None, filter string/tuple, or
	tuple/list of filter strings/tuples
	@keyword use: Optional argument; Use flag(s) telling whether
	to actually perform the action.
	@type use: None, Use flag, or tuple/list of Use flags
        @keyword subtree: Subtree to which to limit the policy, or it
        it already is limited (invariantsubtrees), then additional
        subtrees to consider.
        @type subtree: string or sequence of strings
        @keyword inclusions: C{FileFilter}s to which to limit the policy,
        or if it already is limited (invariantinclusion) then additional
        C{FileFilter}s to include within the general limitation.
        @type inclusions: C{FileFilter} strings, C{FileFilter} tuples,
        or list (not tuple) of C{FileFilter} strings or C{FileFilter} tuples.
	"""
	# enforce abstract base class status
	assert(self.__class__ is not Policy)

	action.RecipeAction.__init__(self, None, [], **keywords)
	self.recipe = recipe
        if self.rootdir is None and self.filetree:
            self.rootdir = {
                DESTDIR: '%(destdir)s',
                BUILDDIR: '%(builddir)s',
                PACKAGE: '',
            }[self.filetree]


    def postInit(self):
        """
        Hook for initialization that cannot happen until after all
        policies have been loaded into the recipe and thus cannot
        happen in __init__; mainly for policies that need to pass
        information to other policies at initialization time.
        """
        pass

    def updateArgs(self, *args, **keywords):
	"""
	The default way to update a class is to override any provided
	keywords.  Subclasses which have the ability to provide more
	intelligent handling can override this method.  This method
	is invoked automatically by recipe.py when a recipe references
	a policy object.  It acts rather like __init__ except that it
	can meaningfully be called more than once for an object.

	Some keyword arguments (at least C{exceptions} and C{subtrees})
	should be appended rather than replaced.
	"""
	exceptions = keywords.pop('exceptions', None)
	if exceptions:
	    if not self.exceptions:
		self.exceptions = []
            if type(exceptions) in (list, tuple):
                self.exceptions.extend(exceptions)
            else:
	        self.exceptions.append(exceptions)
	subtrees = keywords.pop('subtrees', None)
	if subtrees:
	    if not self.subtrees:
		self.subtrees = []
            if type(subtrees) in (list, tuple):
	        self.subtrees.extend(subtrees)
            else:
	        self.subtrees.append(subtrees)

	inclusions = keywords.pop('inclusions', [])
	if (args or inclusions) and not self.inclusions:
            self.inclusions = []

        if inclusions:
            if type(inclusions) == list:
                self.inclusions.extend(inclusions)
            else:
                self.inclusions.append(inclusions)

        if args:
	    self.inclusions.extend(args)

	self.addArgs(**keywords)

    def filterExpression(self, expression, name=None):
        """
        backwards compatibility
        """
        args, kwargs = self.filterExpArgs(expression, name)
        return (args[0],
                args[1],
                kwargs['setmode'],
                kwargs['unsetmode'],
                kwargs['name'],
                kwargs['rootdir'])

    def filterExpArgs(self, expression, name=None):
	"""
	@param expression: regular expression or tuple of
	(regex, [setmode, [unsetmode]])
        Creates arguments to filter.Filter.__init__
        """
        kwargs = {
            'name': name,
            'setmode': None,
            'unsetmode': None,
            'rootdir': self.rootdir,
        }
        macros = self.macros
	if type(expression) in (str, types.FunctionType):
            return (expression, macros), kwargs

	if type(expression) is not list:
	    expression = list(expression)

        # this normally happens when code at a higher level
        # has a filterExp tuple in a list of items and does
        # not need to know how to handle it separately.
        if len(expression) == 1 and type(expression[0]) in (list, tuple):
            expression = list(expression[0])

        regex = expression.pop(0)
        if expression:
            kwargs['setmode'] = expression.pop(0)
        if expression:
            kwargs['unsetmode'] = expression.pop(0)
        return (regex, macros), kwargs

    def compileFilters(self, expressionList, filterList):
        seen = []
	for expression in expressionList:
            if expression in seen:
                # only put each expression on the list once
                continue
            if isinstance(expression, action.Glob):
                expression = expression()
            elif isinstance(expression, action.Regexp):
                expression = expression.pattern
            seen.append(expression)
            args, kwargs = self.filterExpArgs(expression)
            filterList.append(filter.Filter(*args, **kwargs))

    def doProcess(self, recipe):
	"""
	Invocation instance
        @param recipe: holds the recipe object, which is used for
	the macro set and package objects.
        @return: None
        @rtype: None
	"""
	self.recipe = recipe
	self.macros = recipe.macros

        if self.rootdir:
            self.rootdir = self.rootdir % self.macros

        if (hasattr(recipe, '_isDerived')
            and recipe._isDerived == True
            and self.processUnmodified is None):
            # This policy does not handle derived packages
            return

	if hasattr(self.__class__, 'preProcess'):
	    self.preProcess()
{'fraction_non_alphanumeric': 0.07622140503366923, 'fraction_numerical': 0.0048712928029036725, 'mean_word_length': 2.735950044603033, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '475740', 'n_tokens_mistral': 5899, 'n_tokens_neox': 5597, 'n_words': 2281}