<commit_before>import email
import uuid
import boto
from exporters.writers.base_writer import BaseWriter
from exporters.writers.base_writer import ItemsLimitReached
from retrying import retry

ITEMS_PER_BUFFER_WRITE = 5000


class MailWriter(BaseWriter):
    """
    Writes items for email delivery.

    Needed parameters:

        - tmp_folder (str)
            Path to store temp files.

        - email (str)
            Email address where data will be sent.

        - max_size (str)
            maximum size of the attachment for delivery.

        - max_sent (str)
            maximum amount of emails that would be sent.
    """

    parameters = {
        'email': {'type': basestring},
        'max_sent': {'type': int, 'default': 5},
        'aws_login': {'type': basestring},
        'aws_key': {'type': basestring}
    }

    def __init__(self, options, settings):
        super(MailWriter, self).__init__(options, settings)
        self.email = self.read_option('email')
        self.max_sent = self.read_option('sent')
        self.mails_sent = 0
        self.items_per_buffer_write = self.options.get('items_per_buffer_write', ITEMS_PER_BUFFER_WRITE)
        self.ses = boto.connect_ses(self.options['aws_login'], self.options['aws_key'])
        self.items_limit = 10
        self.logger.info('MailWriter has been initiated. Sending to: {}'.format(self.email))

    @retry(wait_exponential_multiplier=500, wait_exponential_max=10000, stop_max_attempt_number=10)
    def write(self, dump_path, group_key=None):
        if self.items_limit or self.max_sent == self.mails_sent:
            raise ItemsLimitReached('Finishing job after items_limit reached: {} items written.'
                                    .format(self.mails_sent))

        m = email.mime.multipart.MIMEMultipart()
        m['Subject'] = 'Test'
        m['From'] = 'Scrapinghub data services <<EMAIL>>'
        m['To'] = self.email

        # Message body
        part = email.mime.text.MIMEText('test file attached')
        m.attach(part)

        # Attachment
        key_name = '{}_{}.{}'.format('ds_dump', uuid.uuid4(), 'gz')
        part = email.mime.text.MIMEText('contents of test file here')
        part.add_header('Content-Disposition', 'attachment; filename=%s' % key_name)
        m.attach(part)

        self.ses.send_raw_email(source=m['From'], raw_message=m.as_string(), destinations=m['To'])
        self.mails_sent += 1
        self.logger.debug('Saved {}'.format(dump_path))<commit_msg>Fixes missing option. Fixes ItemLimitReach issue.<commit_after>import email
import uuid
import boto
from exporters.writers.base_writer import BaseWriter
from exporters.writers.base_writer import ItemsLimitReached
from retrying import retry

ITEMS_PER_BUFFER_WRITE = 5000


class MailWriter(BaseWriter):
    """
    Writes items for email delivery.

    Needed parameters:

        - tmp_folder (str)
            Path to store temp files.

        - email (str)
            Email address where data will be sent.

        - max_size (str)
            maximum size of the attachment for delivery.

        - max_sent (str)
            maximum amount of emails that would be sent.
    """

    parameters = {
        'email': {'type': basestring},
        'max_sent': {'type': int, 'default': 5},
        'aws_login': {'type': basestring},
        'aws_key': {'type': basestring}
    }

    def __init__(self, options, settings):
        super(MailWriter, self).__init__(options, settings)
        self.email = self.read_option('email')
        self.max_sent = self.read_option('max_sent')
        self.mails_sent = 0
        self.items_per_buffer_write = self.options.get('items_per_buffer_write', ITEMS_PER_BUFFER_WRITE)
        self.ses = boto.connect_ses(self.options['aws_login'], self.options['aws_key'])
        self.items_limit = 10
        self.logger.info('MailWriter has been initiated. Sending to: {}'.format(self.email))

    @retry(wait_exponential_multiplier=500, wait_exponential_max=10000, stop_max_attempt_number=10)
    def write(self, dump_path, group_key=None):
        if self.items_limit and self.max_sent == self.mails_sent:
            raise ItemsLimitReached('Finishing job after items_limit reached: {} items written.'
                                    .format(self.mails_sent))

        m = email.mime.multipart.MIMEMultipart()
        m['Subject'] = 'Test'
        m['From'] = 'Scrapinghub data services <<EMAIL>>'
        m['To'] = self.email

        # Message body
        part = email.mime.text.MIMEText('test file attached')
        m.attach(part)

        # Attachment
        key_name = '{}_{}.{}'.format('ds_dump', uuid.uuid4(), 'gz')
        part = email.mime.text.MIMEText('contents of test file here')
        part.add_header('Content-Disposition', 'attachment; filename=%s' % key_name)
        m.attach(part)

        self.ses.send_raw_email(source=m['From'], raw_message=m.as_string(), destinations=m['To'])
        self.mails_sent += 1
        self.logger.debug('Saved {}'.format(dump_path))
{'fraction_non_alphanumeric': 0.12019900497512438, 'fraction_numerical': 0.007960199004975124, 'mean_word_length': 2.7732732732732734, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1860704', 'n_tokens_mistral': 1570, 'n_tokens_neox': 1493, 'n_words': 342}