<commit_before>import os
from datetime import datetime
from django.core.mail import send_mail
from openbudget.settings.base import TEMP_FILES_DIR, ADMINS, EMAIL_HOST_USER
from openbudget.apps.transport.models import String
from openbudget.apps.transport.incoming.parsers import get_parser, get_parser_key


class BaseImporter(object):
    """Gets data out of files and into the database.

    This class is abstract and does not implement the `get_data` method.

    This class can handle any of the supported datasets for importing.

    At this stage, that means Budget Templates (template and related
    node objects), and Budgets/Actuals (Budget or Actual, and related
    items).

    The importer supports a lower level import by parsing the
    file name for meta data - useful while testing, so developers do
    not have to work with an interactive importer. Otherwise, \
    it should be used with an interactive wizard so the content \
    editor can add metadata via a form as part of the import process.

    """

    def __init__(self, sourcefile=None, post_data=None, ignore_unknown_headers=False,
                 ignore_invalid_rows=False, dataset_meta_in_filename=False):

        self.sourcefile = sourcefile
        self.post_data = post_data
        self.ignore_unknown_headers = ignore_unknown_headers
        self.ignore_invalid_rows = ignore_invalid_rows
        self.dataset_meta_in_filename = dataset_meta_in_filename

        if sourcefile:
            self.extract()

    def extract(self):
        """Create a valid dataset from data in the sourcefile.

        We create a dataset object.

        We clean up the headers.

        We email ourselves if the source file cannot be converted \
        to a datset, and we save that sourcefile in a tempoary \
        location, as well as emailing it, for analysis.

        """
        self._extract_meta()

        data_stream = self.sourcefile.read()
        self.data = self.get_data(data_stream)

        return self

    def validate(self):
        """
        Runs validation on the extracted data using the parser's
        validation.
        """
        return self.parser.validate(self.data)

    def save(self):
        """
        Delegates to the parser's save method.
        """
        return self.parser.save()

    def deferred(self):
        """
        Gets a deferred object from the parser, adding to it
        the parser's class key.

        Returns the deferred object which can be later serialized
        and sent to queue, for later resolving it.
        """
        deferred = self.parser.deferred()
        deferred['class'] = get_parser_key(self.parser.__class__)

        return deferred

    def resolve(self, deferred):
        """
        Takes a deferred importer object which sets the appropriate
        parser and delegates to it's resolve method.
        """
        # get the parser class key
        klass = deferred['class']

        if not klass:
            raise Exception('Deferred object missing class key: %s' % klass)

        # get the parser class and resolve the deferred object
        self.parser = get_parser(klass).resolve(deferred)
        # return self for fluency's sake
        return self

    def get_data(self, stream):
        """
        Takes a data stream read from the source file, extracts a
        dataset from it and stores it in `self.data`.
        """
        raise NotImplementedError()

    def normalize_headers(self, headers):
        """Clean the headers of the dataset.

        We replace the existing headers with new ones that \
        have been cleaned and normalized.

        To clean, we strip white space and common joining \
        symbols, and we convert all to lowercase.

        To normalize, we match the header to strings in a \
        string scope map, and convert to the string in the \
        map when our key is either the string or in the scope \
        list.

        """
        symbols = {
            ord('_'): None,
            ord('-'): None,
            ord('"'): None,
            ord(' '): None,
            ord("'"): None,
        }
        scopes_map = self._get_header_scopes()
        normalized_headers = []

        for index, header in enumerate(headers):

            tmp = unicode(header).translate(symbols).lower()

            for k, v in scopes_map.iteritems():

                if tmp == k or tmp in v:
                    normalized_headers[index] = k

        return normalized_headers

    def import_error(self):
        """
        Handle import exceptions by sending an email to admins
        with the file attached.
        """
        #TODO: need to attach the file to the sent email
        dt = datetime.now().isoformat()

        this_file = TEMP_FILES_DIR + \
            '/failed_import_{timestamp}_{filename}'.format(
                timestamp=dt,
                filename=unicode(self.sourcefile)
            )

        with open(this_file, 'wb+') as tmp_file:
            for chunk in self.sourcefile.chunks():
                tmp_file.write(chunk)

        # email ourselves that we have a file to check
        subject = 'Open Budget: Failed File Import'
        message = 'The file is attached.'
        sender = EMAIL_HOST_USER
        recipients = [ADMINS]

        send_mail(subject, message, sender, recipients)
<commit_msg>Fixed import_error method, so that the errored files are correcly attached to the email message.<commit_after>import os
from datetime import datetime
from django.core.mail import EmailMessage
from django.utils.translation import ugettext_lazy as _
from openbudget.settings.base import TEMP_FILES_DIR, ADMINS, EMAIL_HOST_USER
from openbudget.apps.transport.models import String
from openbudget.apps.transport.incoming.parsers import get_parser, get_parser_key


class BaseImporter(object):
    """Gets data out of files and into the database.

    This class is abstract and does not implement the `get_data` method.

    This class can handle any of the supported datasets for importing.

    At this stage, that means Budget Templates (template and related
    node objects), and Budgets/Actuals (Budget or Actual, and related
    items).

    The importer supports a lower level import by parsing the
    file name for meta data - useful while testing, so developers do
    not have to work with an interactive importer. Otherwise, \
    it should be used with an interactive wizard so the content \
    editor can add metadata via a form as part of the import process.

    """

    def __init__(self, sourcefile=None, post_data=None, ignore_unknown_headers=False,
                 ignore_invalid_rows=False, dataset_meta_in_filename=False):

        self.sourcefile = sourcefile
        self.post_data = post_data
        self.ignore_unknown_headers = ignore_unknown_headers
        self.ignore_invalid_rows = ignore_invalid_rows
        self.dataset_meta_in_filename = dataset_meta_in_filename

        if sourcefile:
            self.extract()

    def extract(self):
        """Create a valid dataset from data in the sourcefile.

        We create a dataset object.

        We clean up the headers.

        We email ourselves if the source file cannot be converted \
        to a datset, and we save that sourcefile in a tempoary \
        location, as well as emailing it, for analysis.

        """
        self._extract_meta()

        data_stream = self.sourcefile.read()
        self.data = self.get_data(data_stream)

        return self

    def validate(self):
        """
        Runs validation on the extracted data using the parser's
        validation.
        """
        return self.parser.validate(self.data)

    def save(self):
        """
        Delegates to the parser's save method.
        """
        return self.parser.save()

    def deferred(self):
        """
        Gets a deferred object from the parser, adding to it
        the parser's class key.

        Returns the deferred object which can be later serialized
        and sent to queue, for later resolving it.
        """
        deferred = self.parser.deferred()
        deferred['class'] = get_parser_key(self.parser.__class__)

        return deferred

    def resolve(self, deferred):
        """
        Takes a deferred importer object which sets the appropriate
        parser and delegates to it's resolve method.
        """
        # get the parser class key
        klass = deferred['class']

        if not klass:
            raise Exception('Deferred object missing class key: %s' % klass)

        # get the parser class and resolve the deferred object
        self.parser = get_parser(klass).resolve(deferred)
        # return self for fluency's sake
        return self

    def get_data(self, stream):
        """
        Takes a data stream read from the source file, extracts a
        dataset from it and stores it in `self.data`.
        """
        raise NotImplementedError()

    def normalize_headers(self, headers):
        """Clean the headers of the dataset.

        We replace the existing headers with new ones that \
        have been cleaned and normalized.

        To clean, we strip white space and common joining \
        symbols, and we convert all to lowercase.

        To normalize, we match the header to strings in a \
        string scope map, and convert to the string in the \
        map when our key is either the string or in the scope \
        list.

        """
        symbols = {
            ord('_'): None,
            ord('-'): None,
            ord('"'): None,
            ord(' '): None,
            ord("'"): None,
        }
        scopes_map = self._get_header_scopes()
        normalized_headers = []

        for index, header in enumerate(headers):

            tmp = unicode(header).translate(symbols).lower()

            for k, v in scopes_map.iteritems():

                if tmp == k or tmp in v:
                    normalized_headers[index] = k

        return normalized_headers

    def import_error(self):
        """If a file is not validly formed, we can't open it.

        In this case, we send a copy of the file by email to the site admins.

        We want to site admins to be able to see the file, and understand what \
        was invalid - perhaps it is a problem that can be solved in code - \
        file import is part science, part art.
        """
        dt = datetime.now().isoformat()

        this_file = TEMP_FILES_DIR + \
            '/failed_import_{timestamp}_{filename}'.format(
                timestamp=dt,
                filename=unicode(self.sourcefile)
            )

        with open(this_file, 'wb+') as tmp_file:
            for chunk in self.sourcefile.chunks():
                tmp_file.write(chunk)

        subject = _('Open Budget: Failed File Import')
        message = _('The file is attached.')
        sender = EMAIL_HOST_USER
        recipients = [ADMINS]
        attachment = this_file
        mail = EmailMessage(subject, message, sender, recipients)
        mail.attach_file(attachment)
        mail.send()

{'fraction_non_alphanumeric': 0.07035266528366556, 'fraction_numerical': 0.0, 'mean_word_length': 2.1917098445595853, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5008810', 'n_tokens_mistral': 2925, 'n_tokens_neox': 2793, 'n_words': 1116}