<commit_before>from copy import deepcopy
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _
from openbudget.apps.budgets.models import PATH_SEPARATOR
from openbudget.apps.transport.incoming.errors import DataValidationError


class BaseParser(object):
    """
    Parsers take a raw dataset, usually from an importer, and transform
    it ultimately to persisted data in the datastore.

    This process is combined from 2 steps: validate and save.

    If the two steps are to be divided asynchronously then it's possible to
    get a serializable deferred object after validation, which can later be
    resolved and saved by the same parser class.
    """

    ROUTE_SEPARATOR = PATH_SEPARATOR
    ITEM_SEPARATOR = ';'

    def __init__(self, container_object_dict):
        self.valid = True
        self.dry = False
        self.errors = []
        self.saved_cache = {}
        self.objects_lookup = {}
        # we usually get this from the importer
        self.container_object_dict = container_object_dict

    @classmethod
    def resolve(cls, deferred):
        """
        Resolve a deferred representation of a parser's data.
        Instantiates the class and returns an instance that
        should be ready for saving.
        """
        container_dict = deferred['container']

        if not container_dict:
            raise Exception('Deferred object missing container dict: %s' % container_dict)

        instance = cls(container_dict)
        instance.objects_lookup = deferred['items']

        # basically we're only sure the objects lookup is generated but this
        # should have been side effect of validating the data
        return instance

    def validate(self, data, keep_cache=False):
        """
        Takes a dataset, cleans it, prepares it for saving
        and runs all possible validations to make sure
        that a consecutive call on this prepared data
        sends it straight to the datastore without any exceptions.

        It generate a lookup dictionary of the dataset that will later be
        iterated upon when saving.

        If the `keep_cache` argument is `True` then the parser's `saved_cache`
        property, used for storing item instances created from the dataset, is
        not cleared at the end of the validation.
        You'll have to explicitly call `_clear_cache()` after that when you
        need to.

        Returns a boolean if validation is successful and a list of errors that
        were thrown during validation.
        """
        # generate a lookup table with each item uniquely identified
        self._generate_lookup(data)

        self.keep_cache = keep_cache
        # run a dry save of the data
        self.save(dry=True)

        return self.valid, self.errors

    def save(self, dry=False):
        """
        Saves the pre-stored objects in the `objects_lookup` dict, as items.
        Also creates the container from the data given on instantiation.

        When running in dry mode nothing is actually saved to the datastore
        and nothing is persisted.
        All the generated model instances are stored in the `saved_cache` dict
        and never saved to datastore.
        """
        self.dry = dry

        # create an instance of the container
        self._create_container()

        if dry:
            # save an untempered copy
            lookup_table_copy = deepcopy(self.objects_lookup)

        # loop the lookup table and save every item
        for key, obj in self.objects_lookup.iteritems():
            self._save_item(obj, key)

        if dry:
            if not self.keep_cache:
                self._clear_cache()
            # clear all changes by replacing the lookup with the old copy
            self.objects_lookup = lookup_table_copy

        self.dry = False

        return True

    def deferred(self):
        """
        Generates and returns a deferredable representation of the
        parser.
        """
        return {
            'container': self.container_object_dict,
            'items': self.objects_lookup
        }

    def throw(self, error):
        """
        Takes an error instance from the `incoming.errors` module and
        appends it to the list of errors.
        Also makes sure `valid` is `False`.
        """
        self.valid = False
        self.errors.append(error)
        return self

    def _generate_lookup(self, data):
        """
        Generates the data's objects lookup table for saving them later.

        This method needs to be implemented by non-abstract implementations.
        """
        raise NotImplementedError

    def _save_item(self, obj, key):
        """
        Saves a given object as an item - as in instance of `item_model` attribute.

        This method needs to be implemented by non-abstract implementations.
        """
        raise NotImplementedError

    def _clean_object(self, obj, key):
        """
        Cleans an object before it's used as attributes for an item's
        model instance creation.
        Removes every attribute that is not in the `ITEM_ATTRIBUTES` dict.
        """
        for attr in obj:
            if attr not in self.ITEM_ATTRIBUTES:
                del obj[attr]

    def _create_container(self, container_dict=None, exclude=None):
        """
        Creates a model instance of the `container_model`.
        """
        data = container_dict or self.container_object_dict

        if not self.dry:
            container = self.container_model.objects.create(**data)
        else:
            container = self.container_model(**data)
            self._dry_clean(container, exclude=exclude)

        self.container_object = container

    def _dry_clean(self, instance, row_num=None, exclude=None):
        try:
            instance.full_clean(exclude=exclude)
        except ValidationError as e:
            self.throw(
                DataValidationError(reasons=e.message_dict, row=row_num)
            )

    def _clear_cache(self):
        self.saved_cache.clear()


PARSERS_MAP = {}


def register(key, parser_class):
    PARSERS_MAP[key] = parser_class


def get_parser(key):
    if key in PARSERS_MAP:
        return PARSERS_MAP[key]
    else:
        raise Exception(_('Parser for key: "%s" does not exist') % key)


def get_parser_key(cls):
    for key, parser_class in PARSERS_MAP.iteritems():
        if cls is parser_class:
            return key
    else:
        raise Exception(_('Given parser is not registered: %s' % unicode(cls)))


def autodiscover():<commit_msg>Finished docs<commit_after>from copy import deepcopy
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _
from openbudget.apps.budgets.models import PATH_SEPARATOR
from openbudget.apps.transport.incoming.errors import DataValidationError


class BaseParser(object):
    """
    Parsers take a raw dataset, usually from an importer, and transform
    it ultimately to persisted data in the datastore.

    This process is combined from 2 steps: validate and save.

    If the two steps are to be divided asynchronously then it's possible to
    get a serializable deferred object after validation, which can later be
    resolved and saved by the same parser class.
    """

    ROUTE_SEPARATOR = PATH_SEPARATOR
    ITEM_SEPARATOR = ';'

    def __init__(self, container_object_dict):
        self.valid = True
        self.dry = False
        self.errors = []
        self.saved_cache = {}
        self.objects_lookup = {}
        # we usually get this from the importer
        self.container_object_dict = container_object_dict

    @classmethod
    def resolve(cls, deferred):
        """
        Resolve a deferred representation of a parser's data.
        Instantiates the class and returns an instance that
        should be ready for saving.
        """
        container_dict = deferred['container']

        if not container_dict:
            raise Exception('Deferred object missing container dict: %s' % container_dict)

        instance = cls(container_dict)
        instance.objects_lookup = deferred['items']

        # basically we're only sure the objects lookup is generated but this
        # should have been side effect of validating the data
        return instance

    def validate(self, data, keep_cache=False):
        """
        Takes a dataset, cleans it, prepares it for saving
        and runs all possible validations to make sure
        that a consecutive call on this prepared data
        sends it straight to the datastore without any exceptions.

        It generate a lookup dictionary of the dataset that will later be
        iterated upon when saving.

        If the `keep_cache` argument is `True` then the parser's `saved_cache`
        property, used for storing item instances created from the dataset, is
        not cleared at the end of the validation.
        You'll have to explicitly call `_clear_cache()` after that when you
        need to.

        Returns a boolean if validation is successful and a list of errors that
        were thrown during validation.
        """
        # generate a lookup table with each item uniquely identified
        self._generate_lookup(data)

        self.keep_cache = keep_cache
        # run a dry save of the data
        self.save(dry=True)

        return self.valid, self.errors

    def save(self, dry=False):
        """
        Saves the pre-stored objects in the `objects_lookup` dict, as items.
        Also creates the container from the data given on instantiation.

        When running in dry mode nothing is actually saved to the datastore
        and nothing is persisted.
        All the generated model instances are stored in the `saved_cache` dict
        and never saved to datastore.
        """
        self.dry = dry

        # create an instance of the container
        self._create_container()

        if dry:
            # save an untempered copy
            lookup_table_copy = deepcopy(self.objects_lookup)

        # loop the lookup table and save every item
        for key, obj in self.objects_lookup.iteritems():
            self._save_item(obj, key)

        if dry:
            if not self.keep_cache:
                self._clear_cache()
            # clear all changes by replacing the lookup with the old copy
            self.objects_lookup = lookup_table_copy

        self.dry = False

        return True

    def deferred(self):
        """
        Generates and returns a deferredable representation of the
        parser.
        """
        return {
            'container': self.container_object_dict,
            'items': self.objects_lookup
        }

    def throw(self, error):
        """
        Takes an error instance from the `incoming.errors` module and
        appends it to the list of errors.
        Also makes sure `valid` is `False`.
        """
        self.valid = False
        self.errors.append(error)
        return self

    def _generate_lookup(self, data):
        """
        Generates the data's objects lookup table for saving them later.

        This method needs to be implemented by non-abstract implementations.
        """
        raise NotImplementedError

    def _save_item(self, obj, key):
        """
        Saves a given object as an item - as in instance of `item_model` attribute.

        This method needs to be implemented by non-abstract implementations.
        """
        raise NotImplementedError

    def _clean_object(self, obj, key):
        """
        Cleans an object before it's used as attributes for an item's
        model instance creation.
        Removes every attribute that is not in the `ITEM_ATTRIBUTES` dict.
        """
        for attr in obj:
            if attr not in self.ITEM_ATTRIBUTES:
                del obj[attr]

    def _create_container(self, container_dict=None, exclude=None):
        """
        Creates a model instance of the `container_model`.
        """
        data = container_dict or self.container_object_dict

        if not self.dry:
            container = self.container_model.objects.create(**data)
        else:
            container = self.container_model(**data)
            self._dry_clean(container, exclude=exclude)

        self.container_object = container

    def _dry_clean(self, instance, row_num=None, exclude=None):
        """
        Calls a given `instance`'s `full_clean()` method for validating it.
        Validation errors are caught and exchanged for errors thrown to the
        parser using `throw()`.
        """
        try:
            instance.full_clean(exclude=exclude)
        except ValidationError as e:
            self.throw(
                DataValidationError(reasons=e.message_dict, row=row_num)
            )

    def _clear_cache(self):
        """
        Clears the `saved_cache` dict.
        """
        self.saved_cache.clear()


PARSERS_MAP = {}


def register(key, parser_class):
    """
    Registers a parser class in the system.
    """
    PARSERS_MAP[key] = parser_class


def get_parser(key):
    """
    Gets a parser class from the registry using a string key.
    """
    if key in PARSERS_MAP:
        return PARSERS_MAP[key]
    else:
        raise Exception(_('Parser for key: "%s" does not exist') % key)


def get_parser_key(cls):
    """
    Gets the key from the registry under which this parser class
    is stored, mostly for deferring that parser for later use.
    """
    for key, parser_class in PARSERS_MAP.iteritems():
        if cls is parser_class:
            return key
    else:
        raise Exception(_('Given parser is not registered: %s' % unicode(cls)))


def autodiscover():
{'fraction_non_alphanumeric': 0.06516837588188232, 'fraction_numerical': 0.00014546512473634447, 'mean_word_length': 2.27069457659372, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3397636', 'n_tokens_mistral': 3630, 'n_tokens_neox': 3454, 'n_words': 1354}