<commit_before># Miscellaneous utility functions and classes found here.
import collections
import dbm
import re
import shelve
import json
from io import BytesIO
from pickle import Pickler, Unpickler
from rs_errors import CommandSyntaxError
import argparse


class DotDict(dict):
    """
    Custom dictionary format that allows member access by using dot notation:
    eg - dict.key.subkey
    """

    def __init__(self, d, **kwargs):
        super().__init__(**kwargs)
        for k, v in d.items():
            if isinstance(v, collections.Mapping):
                v = DotDict(v)
            self[k] = v

    def __getattr__(self, item):
        try:
            return super().__getitem__(item)
        except KeyError as e:
            raise AttributeError(str(e)) from None

    def __setattr__(self, key, value):
        if isinstance(value, collections.Mapping):
            value = DotDict(value)
        super().__setitem__(key, value)

    __delattr__ = dict.__delitem__


class Cupboard(shelve.Shelf):
    """
    Custom Shelf implementation that only pickles values at save-time.
    Increases save/load times, decreases get/set item times.
    More suitable for use as a savable dictionary.
    """
    def __init__(self, filename, flag='c', protocol=None, keyencoding='utf-8'):
        self.db = filename
        self.flag = flag
        self.dict = {}
        with dbm.open(self.db, self.flag) as db:
            for k in db.keys():
                v = BytesIO(db[k])
                try:
                    self.dict[k] = Unpickler(v).load()
                except ModuleNotFoundError:  # Just throw it away if it won't load.
                    del db[k]
        shelve.Shelf.__init__(self, self.dict, protocol, False, keyencoding)

    def __getitem__(self, key):
        return self.dict[key.encode(self.keyencoding)]

    def __setitem__(self, key, value):
        self.dict[key.encode(self.keyencoding)] = value

    def __delitem__(self, key):
        del self.dict[key.encode(self.keyencoding)]

    def sync(self):
        with dbm.open(self.db, self.flag) as db:
            for k, v in self.dict.items():
                f = BytesIO()
                p = Pickler(f, protocol=self._protocol)
                p.dump(v)
                db[k] = f.getvalue()
            db.sync()

    def close(self):
        try:
            self.sync()
        finally:
            try:
                self.dict = shelve._ClosedDict()
            except Exception:
                self.dict = None


class RSNamespace(argparse.Namespace):
    def __getitem__(self, key):
        try:
            return self.__getattribute__(key)
        except AttributeError:
            raise KeyError

    def __setitem__(self, key, value):
        self.__setattr__(key, value)


class RSArgumentParser(argparse.ArgumentParser):

    def __init__(self, add_help=False, ignore_unrecognized_arguments=True, **kwargs):
        self.ignore_unrecognized_arguments = ignore_unrecognized_arguments
        super().__init__(self, add_help=add_help, **kwargs)

    def exit(self, status=0, message=None):
        raise CommandSyntaxError(message)

    def error(self, message):
        """error(message: string)

        Prints a usage message incorporating the message to stderr and
        exits.

        If you override this in a subclass, it should not return -- it
        should either exit or raise an exception.
        """
        # self.print_usage(_sys.stderr)
        # args = {'prog': self.prog, 'message': message}
        # self.exit(2, _('%(prog)s: error: %(message)s\n') % args)
        raise CommandSyntaxError(message)

    def parse_args(self, args=None, namespace=None):
        args, argv = self.parse_known_args(args, namespace)
        if argv and not self.ignore_unrecognized_arguments:
            msg = 'Unrecognized arguments: %s'
            self.error(msg % ' '.join(argv))
        return args

    def parse_known_args(self, args=None, namespace=None):
        if args is None:
            raise CommandSyntaxError("No arguments. How did you even manage that.")
        else:
            # make sure that args are mutable
            args = list(args)

        # default Namespace built from parser defaults
        if namespace is None:
            namespace = RSNamespace()
<commit_msg>!Purge --verbose now also shows author of the message and their ID<commit_after># Miscellaneous utility functions and classes found here.
import collections
import dbm
import re
import shelve
import json
from io import BytesIO
from pickle import Pickler, Unpickler
from rs_errors import CommandSyntaxError
import argparse


class DotDict(dict):
    """
    Custom dictionary format that allows member access by using dot notation:
    eg - dict.key.subkey
    """

    def __init__(self, d, **kwargs):
        super().__init__(**kwargs)
        for k, v in d.items():
            if isinstance(v, collections.Mapping):
                v = DotDict(v)
            self[k] = v

    def __getattr__(self, item):
        try:
            return super().__getitem__(item)
        except KeyError as e:
            raise AttributeError(str(e)) from None

    def __setattr__(self, key, value):
        if isinstance(value, collections.Mapping):
            value = DotDict(value)
        super().__setitem__(key, value)

    __delattr__ = dict.__delitem__


class Cupboard(shelve.Shelf):
    """
    Custom Shelf implementation that only pickles values at save-time.
    Increases save/load times, decreases get/set item times.
    More suitable for use as a savable dictionary.
    """
    def __init__(self, filename, flag='c', protocol=None, keyencoding='utf-8'):
        self.db = filename
        self.flag = flag
        self.dict = {}
        with dbm.open(self.db, self.flag) as db:
            for k in db.keys():
                v = BytesIO(db[k])
                try:
                    self.dict[k] = Unpickler(v).load()
                except ModuleNotFoundError:  # Just throw it away if it won't load.
                    del db[k]
        shelve.Shelf.__init__(self, self.dict, protocol, False, keyencoding)

    def __getitem__(self, key):
        return self.dict[key.encode(self.keyencoding)]

    def __setitem__(self, key, value):
        self.dict[key.encode(self.keyencoding)] = value

    def __delitem__(self, key):
        del self.dict[key.encode(self.keyencoding)]

    def sync(self):
        with dbm.open(self.db, self.flag) as db:
            for k, v in self.dict.items():
                f = BytesIO()
                p = Pickler(f, protocol=self._protocol)
                p.dump(v)
                db[k] = f.getvalue()
            db.sync()

    def close(self):
        try:
            self.sync()
        finally:
            try:
                self.dict = shelve._ClosedDict()
            except Exception:
                self.dict = None


class RSNamespace(argparse.Namespace):
    def __getitem__(self, key):
        try:
            return self.__getattribute__(key)
        except AttributeError:
            raise KeyError

    def __setitem__(self, key, value):
        self.__setattr__(key, value)


class RSArgumentParser(argparse.ArgumentParser):

    def __init__(self, add_help=False, ignore_unrecognized_arguments=True, **kwargs):
        self.ignore_unrecognized_arguments = ignore_unrecognized_arguments
        super().__init__(self, add_help=add_help, **kwargs)

    def exit(self, status=0, message=None):
        raise CommandSyntaxError(message)

    def error(self, message):
        raise CommandSyntaxError(message)

    def parse_args(self, args=None, namespace=None):
        args, argv = self.parse_known_args(args, namespace)
        if argv and not self.ignore_unrecognized_arguments:
            msg = 'Unrecognized arguments: %s'
            self.error(msg % ' '.join(argv))
        return args

    def parse_known_args(self, args=None, namespace=None):
        if args is None:
            raise CommandSyntaxError("No arguments. How did you even manage that.")
        else:
            # make sure that args are mutable
            args = list(args)

        # default Namespace built from parser defaults
        if namespace is None:
            namespace = RSNamespace()

{'fraction_non_alphanumeric': 0.0962936046511628, 'fraction_numerical': 0.0006056201550387597, 'mean_word_length': 2.214091086025691, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5468165', 'n_tokens_mistral': 2374, 'n_tokens_neox': 2272, 'n_words': 672}