<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2010-2012 <NAME>
# This software is distributed under the two-clause BSD license.

from __future__ import unicode_literals, absolute_import

"""Handle repository-wide configuration.

These settings only relate to the list of categories, files, ... registered in
the repository, regardless of current settings.
"""


import fnmatch
import os
import socket
import stat
import time

import confutils

from . import action_parser
from . import actions
from . import constants
from . import fs
from . import helpers
from . import rule_parser


class FileConfig(object):
    """Definition of the action for a file."""

    COPY = 'copy'
    SYMLINK = 'symlink'
    PARSE = 'parse'

    ACTIONS = {
        COPY: actions.CopyAction,
        SYMLINK: actions.SymLinkAction,
        PARSE: actions.FileProcessingAction,
    }

    def __init__(self, action, **options):
        if action not in self.ACTIONS:
            raise ValueError("Invalid action %s, choose one of %s" % (
                action, ', '.join(self.ACTIONS)))
        self.action = action
        self.options = options

    def get_destination(self, filename, target=''):
        """Find the appropriate target for a file."""
        if 'dest' in self.options:
            # Explicit target path
            destination = self.options['dest']
        elif 'destdir' in self.options:
            # Replaced destination folder
            destination = os.path.join(self.options['destdir'],
                os.path.basename(filename))
        else:
            destination = filename

        return helpers.get_absolute_path(destination, base=target)

    def get_action(self, filename, env):
        action = self.ACTIONS[self.action]
        abs_source = helpers.get_absolute_path(filename, base=env.root)
        abs_dest = self.get_destination(filename, env.target)

        return action(source=abs_source, destination=abs_dest,
            env=env, **self.options)

    def __repr__(self):
        return '<FileConfig: %s %r>' % (self.action, self.options)


class GlobbingDict(dict):
    def __getitem__(self, key):
        for glob, value in self.items():
            if fnmatch.fnmatchcase(key, glob):
                return value
        raise KeyError("Key %s not found in %r" % (key, self))

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __repr__(self):
        return "GlobbingDict(%s)" % super(GlobbingDict, self).__repr__()


class RepositoryView(object):
    """A repository, as seen for a given set of initial categories.

    Attributes:
        base: the Repository on which this view is based
        categories: frozenset of active category names
    """

    def __init__(self, base):
        self.base = base
        self.categories = frozenset()

    def set_initial_categories(self, initial):
        self.categories = frozenset(initial)
        for category_rule, extra_categories in self.base.category_rules:
            if category_rule.test(self.categories):
                self.categories |= extra_categories

    def iter_files(self):
        """Retrieve all active files for this view

        Yields:
            filename
        """
        for file_rule, filename in self.base.file_rules:
            if file_rule.test(self.categories):
                yield filename

    def get_file_config(self, filename, default_action='parse'):
        default_config = FileConfig(default_action)
        return self.base.file_configs.get(filename, default_config)


class Repository(object):
    """Holds repository configuration.

    Attributes:
        categories (str set): all categories
        files (str list): all managed files
        file_actions (dict(str => FileConfig)): actions for files
        rule_lexer (rule_parser.RuleLexer): lexer to use for rule parsing
    """

    def __init__(self, root=None, *args, **kwargs):
        self.root = root
        self.config = confutils.ConfigFile()
        self.actions_config = self.config.section_view('actions')
        self.files_config = self.config.section_view('files', True)
        self.categories_config = self.config.section_view('categories', True)

        self.category_rules = []
        self.file_rules = []
        self.file_configs = GlobbingDict()
        self.rule_lexer = rule_parser.RuleLexer()
        self.action_lexer = action_parser.ActionLexer()

        self._read_config()
<commit_msg>Improve [actions] handling.<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2010-2012 <NAME>
# This software is distributed under the two-clause BSD license.

from __future__ import unicode_literals, absolute_import

"""Handle repository-wide configuration.

These settings only relate to the list of categories, files, ... registered in
the repository, regardless of current settings.
"""


import fnmatch
import os
import socket
import stat
import time

import confutils

from . import action_parser
from . import actions
from . import constants
from . import fs
from . import helpers
from . import rule_parser


class FileConfig(object):
    """Definition of the action for a file."""

    COPY = 'copy'
    SYMLINK = 'symlink'
    PARSE = 'parse'

    ACTIONS = {
        COPY: actions.CopyAction,
        SYMLINK: actions.SymLinkAction,
        PARSE: actions.FileProcessingAction,
    }

    def __init__(self, action, **options):
        if action not in self.ACTIONS:
            raise ValueError("Invalid action %s, choose one of %s" % (
                action, ', '.join(self.ACTIONS)))
        self.action = action
        self.options = options

    def get_destination(self, filename, target=''):
        """Find the appropriate target for a file."""
        if 'dest' in self.options:
            # Explicit target path
            destination = self.options['dest']
        elif 'destdir' in self.options:
            # Replaced destination folder
            destination = os.path.join(self.options['destdir'],
                os.path.basename(filename))
        else:
            destination = filename

        return helpers.get_absolute_path(destination, base=target)

    def get_action(self, filename, env):
        action = self.ACTIONS[self.action]
        abs_source = helpers.get_absolute_path(filename, base=env.root)
        abs_dest = self.get_destination(filename, env.target)

        return action(source=abs_source, destination=abs_dest,
            env=env, **self.options)

    def __repr__(self):
        return '<FileConfig: %s %r>' % (self.action, self.options)


class GlobStore(object):
    def __init__(self, *args):
        self.entries = list(*args)

    def __getitem__(self, key):
        for glob, value in self.entries:
            if fnmatch.fnmatchcase(key, glob):
                return value
        raise KeyError("Key %s not found in %r" % (key, self))

    def __setitem__(self, key, value):
        for i, entry in enumerate(self.entries):
            if key == entry[0]:
                self.entries[i] = (key, value)
        self.entries.append((key, value))

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __repr__(self):
        return "GlobStore(%r)" % self.entries


class RepositoryView(object):
    """A repository, as seen for a given set of initial categories.

    Attributes:
        base: the Repository on which this view is based
        categories: frozenset of active category names
    """

    def __init__(self, base):
        self.base = base
        self.categories = frozenset()

    def set_initial_categories(self, initial):
        self.categories = frozenset(initial)
        for category_rule, extra_categories in self.base.category_rules:
            if category_rule.test(self.categories):
                self.categories |= extra_categories

    def iter_files(self):
        """Retrieve all active files for this view

        Yields:
            filename
        """
        for file_rule, filename in self.base.file_rules:
            if file_rule.test(self.categories):
                yield filename

    def get_file_config(self, filename, default_action='parse'):
        default_config = FileConfig(default_action)
        return self.base.file_configs.get(filename, default_config)


class Repository(object):
    """Holds repository configuration.

    Attributes:
        categories (str set): all categories
        files (str list): all managed files
        file_actions (dict(str => FileConfig)): actions for files
        rule_lexer (rule_parser.RuleLexer): lexer to use for rule parsing
    """

    def __init__(self, root=None, *args, **kwargs):
        self.root = root
        self.config = confutils.ConfigFile()
        self.actions_config = self.config.section_view('actions')
        self.files_config = self.config.section_view('files', True)
        self.categories_config = self.config.section_view('categories', True)

        self.category_rules = []
        self.file_rules = []
        self.file_configs = GlobStore()
        self.rule_lexer = rule_parser.RuleLexer()
        self.action_lexer = action_parser.ActionLexer()

        self._read_config()

{'fraction_non_alphanumeric': 0.09773706896551725, 'fraction_numerical': 0.0020474137931034485, 'mean_word_length': 2.5764932562620424, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6669789', 'n_tokens_mistral': 2605, 'n_tokens_neox': 2509, 'n_words': 733}