<commit_before># -*- coding: utf-8 -*-

import re
import sys

try:
    import imbox
except ImportError:
    imbox = None
import requests

from intelmq.lib.bot import Bot
from intelmq.lib.message import Report


class MailURLCollectorBot(Bot):

    def init(self):
        if imbox is None:
            self.logger.error('Could not import imbox. Please install it.')
            self.stop()

    def process(self):
        mailbox = imbox.Imbox(self.parameters.mail_host,
                              self.parameters.mail_user,
                              self.parameters.mail_password,
                              self.parameters.mail_ssl)
        emails = mailbox.messages(folder=self.parameters.folder, unread=True)

        if emails:
            for uid, message in emails:

                if (self.parameters.subject_regex and
                        not re.search(self.parameters.subject_regex,
                                      message.subject)):
                    continue

                self.logger.info("Reading email report")

                for body in message.body['plain']:
                    match = re.search(self.parameters.url_regex, str(body))
                    if match:
                        url = match.group()

                        # Build request
                        self.http_header = getattr(self.parameters,
                                'http_header', {})
                        self.http_verify_cert = getattr(self.parameters,
                                                        'http_verify_cert', True)

                        if hasattr(self.parameters, 'http_user') and hasattr(
                                self.parameters, 'http_password'):
                            self.auth = (self.parameters.http_user,
                                         self.parameters.http_password)
                        else:
                            self.auth = None

                        http_proxy = getattr(self.parameters, 'http_proxy', None)
                        https_proxy = getattr(self.parameters,
                                              'http_ssl_proxy', None)
                        if http_proxy and https_proxy:
                            self.proxy = {'http': http_proxy, 'https': https_proxy}<commit_msg>Strip certain characters from URLs<commit_after># -*- coding: utf-8 -*-

import re
import sys

try:
    import imbox
except ImportError:
    imbox = None
import requests

from intelmq.lib.bot import Bot
from intelmq.lib.message import Report


class MailURLCollectorBot(Bot):

    def init(self):
        if imbox is None:
            self.logger.error('Could not import imbox. Please install it.')
            self.stop()

    def process(self):
        mailbox = imbox.Imbox(self.parameters.mail_host,
                              self.parameters.mail_user,
                              self.parameters.mail_password,
                              self.parameters.mail_ssl)
        emails = mailbox.messages(folder=self.parameters.folder, unread=True)

        if emails:
            for uid, message in emails:

                if (self.parameters.subject_regex and
                        not re.search(self.parameters.subject_regex,
                                      message.subject)):
                    continue

                self.logger.info("Reading email report")

                for body in message.body['plain']:
                    match = re.search(self.parameters.url_regex, str(body))
                    if match:
                        url = match.group()
                        url = url.strip()     # strip leading and trailing spaces
                        url = url.strip('\n') # strip newlines
                        url = url.strip('\r') # strip carriage returns

                        # Build request
                        self.http_header = getattr(self.parameters,
                                'http_header', {})
                        self.http_verify_cert = getattr(self.parameters,
                                                        'http_verify_cert', True)

                        if hasattr(self.parameters, 'http_user') and hasattr(
                                self.parameters, 'http_password'):
                            self.auth = (self.parameters.http_user,
                                         self.parameters.http_password)
                        else:
                            self.auth = None

                        http_proxy = getattr(self.parameters, 'http_proxy', None)
                        https_proxy = getattr(self.parameters,
                                              'http_ssl_proxy', None)
                        if http_proxy and https_proxy:
                            self.proxy = {'http': http_proxy, 'https': https_proxy}
{'fraction_non_alphanumeric': 0.07851667702506733, 'fraction_numerical': 0.0004143360265175057, 'mean_word_length': 1.1846153846153846, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7144315', 'n_tokens_mistral': 1124, 'n_tokens_neox': 1065, 'n_words': 253}