<commit_before># -*- coding: utf-8 -*-
import io
import re
import requests
import sys
import zipfile

from intelmq.lib.bot import Bot
from intelmq.lib.message import Report

import rt


class RTCollectorBot(Bot):

    def init(self):
        self.http_header = getattr(self.parameters, 'http_header', {})
        self.http_verify_cert = getattr(self.parameters, 'http_verify_cert',
                                        True)

        http_proxy = getattr(self.parameters, 'http_proxy', None)
        https_proxy = getattr(self.parameters, 'http_ssl_proxy', None)
        if http_proxy and https_proxy:
            self.proxy = {'http': http_proxy, 'https': https_proxy}
        else:
            self.proxy = None

        self.http_header['User-agent'] = self.parameters.http_user_agent

    def process(self):
        RT = rt.Rt(self.parameters.uri, self.parameters.user,
                   self.parameters.password)
        if not RT.login():
            raise ValueError('Login failed.')

        query = RT.search(Queue=self.parameters.search_queue,
                          Subject__like=self.parameters.search_subject_like,
                          Owner=self.parameters.search_owner,
                          Status=self.parameters.search_status)
        self.logger.info('{} results on search query.'.format(len(query)))

        for ticket in query:
            ticket_id = int(ticket['id'].split('/')[1])
            self.logger.debug('Process ticket {}.'.format(ticket_id))
            content = 'attachment'
            for (att_id, att_name, _, _) in RT.get_attachments(ticket_id):
                if re.search(self.parameters.attachment_regex, att_name):
                    self.logger.debug('Found attachment {}: {!r}.'
                                      ''.format(att_id, att_name))
                    break
            else:
                ticket = RT.get_history(ticket_id)[0]
                text = ticket['Content']
                created = ticket['Created']
                urlmatch = re.search(self.parameters.url_regex, text)
                if urlmatch:
                    content = 'url'
                    url = urlmatch.group(0)
                else:
                    self.logger.debug('No matching attachment or URL found.')
                    continue
            if content == 'attachment':
                attachment = RT.get_attachment_content(ticket_id, att_id)
                created = RT.get_attachment(ticket_id, att_id)['Created']

                if self.parameters.unzip_attachment:
                    file_obj = io.BytesIO(attachment)
                    zipped = zipfile.ZipFile(file_obj)<commit_msg>ENH: RT collector: sort by creation date<commit_after># -*- coding: utf-8 -*-
import io
import re
import requests
import sys
import zipfile

from intelmq.lib.bot import Bot
from intelmq.lib.message import Report

import rt


class RTCollectorBot(Bot):

    def init(self):
        self.http_header = getattr(self.parameters, 'http_header', {})
        self.http_verify_cert = getattr(self.parameters, 'http_verify_cert',
                                        True)

        http_proxy = getattr(self.parameters, 'http_proxy', None)
        https_proxy = getattr(self.parameters, 'http_ssl_proxy', None)
        if http_proxy and https_proxy:
            self.proxy = {'http': http_proxy, 'https': https_proxy}
        else:
            self.proxy = None

        self.http_header['User-agent'] = self.parameters.http_user_agent

    def process(self):
        RT = rt.Rt(self.parameters.uri, self.parameters.user,
                   self.parameters.password)
        if not RT.login():
            raise ValueError('Login failed.')

        query = RT.search(Queue=self.parameters.search_queue,
                          Subject__like=self.parameters.search_subject_like,
                          Owner=self.parameters.search_owner,
                          Status=self.parameters.search_status,
                          order='Created')
        self.logger.info('{} results on search query.'.format(len(query)))

        for ticket in query:
            ticket_id = int(ticket['id'].split('/')[1])
            self.logger.debug('Process ticket {}.'.format(ticket_id))
            content = 'attachment'
            for (att_id, att_name, _, _) in RT.get_attachments(ticket_id):
                if re.search(self.parameters.attachment_regex, att_name):
                    self.logger.debug('Found attachment {}: {!r}.'
                                      ''.format(att_id, att_name))
                    break
            else:
                ticket = RT.get_history(ticket_id)[0]
                text = ticket['Content']
                created = ticket['Created']
                urlmatch = re.search(self.parameters.url_regex, text)
                if urlmatch:
                    content = 'url'
                    url = urlmatch.group(0)
                else:
                    self.logger.debug('No matching attachment or URL found.')
                    continue
            if content == 'attachment':
                attachment = RT.get_attachment_content(ticket_id, att_id)
                created = RT.get_attachment(ticket_id, att_id)['Created']

                if self.parameters.unzip_attachment:
                    file_obj = io.BytesIO(attachment)
                    zipped = zipfile.ZipFile(file_obj)
{'fraction_non_alphanumeric': 0.10303928836174944, 'fraction_numerical': 0.0014825796886582653, 'mean_word_length': 1.7676923076923077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '357933', 'n_tokens_mistral': 1462, 'n_tokens_neox': 1391, 'n_words': 290}