<commit_before>"""
Reporters module.

There are several classes for specific format reporting, and also some of the
slack conversation/convention parsers.
"""
import os
import errno
import html.parser
import logging
import re

from slack_backup import objects as o
from slack_backup import utils
from slack_backup import emoji


class Reporter(object):
    """Base reporter class"""
    ext = ''
    symbols = {'plain': {'join': '->',
                         'leave': '<-',
                         'me': '*',
                         'file': '-',
                         'topic': '+',
                         'separator': '|'},
               'unicode': {'join': '⮊',
                           'leave': '⮈',
                           'me': '🟊',
                           'file': '📂',
                           'topic': '🟅',
                           'separator': '│'}}
    url_pat = re.compile(r'(?P<replace><http[^>]+>)')
    slackid_pat = re.compile(r'(?P<replace><@'
                             '(?P<slackid>U[A-Z,0-9]+)(\|[^>]+)?[^>]*>)')

    def __init__(self, args, query):
        self.out = args.output
        self.theme = args.theme
        self.q = query
        self.types = {"channel_join": self._msg_join,
                      "channel_leave": self._msg_leave,
                      "channel_topic": self._msg_topic,
                      "file_share": self._msg_file,
                      "me_message": self._msg_me}

        self.emoji = emoji.EMOJI.get(args.theme, {})

        self.channels = self._get_channels(args.channels)
        self.users = self.q(o.User).all()

    def generate(self):
        """Generate raport for each channel"""
        for channel in self.channels:
            messages = []
            log_path = self.get_log_path(channel.name)
            self._set_max_len(channel)
            try:
                os.unlink(log_path)
            except IOError as err:
                if err.errno != errno.ENOENT:
                    raise
            for message in self.q(o.Message).\
                    filter(o.Message.channel == channel).\
                    order_by(o.Message.ts).all():
                messages.append(message)
            self.write_msg(messages, log_path)

    def get_log_path(self, name):
        """Return relative log file name """
        return os.path.join(self.out, name + self.ext)

    def write_msg(self, messages, log):
        """Write message to file"""
        with open(log, "a") as fobj:
            for message in messages:
                data = self._process_message(message)
                fobj.write(data['tpl'].format(**data))

    def _get_symbol(self, item):
        """Return appropriate item depending on the selected theme"""
        return self.symbols[self.theme][item]

    def _get_channels(self, selected_channels):
        """
        Retrieve channels from db and return those which names matched from
        selected_channels list
        """
        all_channels = self.q(o.Channel).all()
        if not selected_channels:
            return all_channels

        result = []
        for channel in all_channels:
            if channel.name in selected_channels:
                result.append(channel)
        return result

    def _process_message(self, msg):
        """
        Make changes to the text (replace slack ids, replace representation of
        urls, substitute images etc) and return dict with data suitable to
        display.
        """
        processor = self.types.get(msg.type, self._msg)
        data = processor(msg)
        data.update({'date': msg.datetime().strftime("%Y-%m-%d %H:%M:%S"),
                'tpl': "{date} {nick} {msg}"})

        return data

    def _msg_join(self, msg):
        """return data for join"""
        return {'msg': msg.text,
                'nick': self._get_symbol('join')}

    def _msg_leave(self, msg):
        """return data for leave"""
        return {'msg': msg.text,
                'nick': self._get_symbol('leave')}

    def _msg_topic(self, msg):
        """return data for set topic"""
        return {'msg': msg.text,
                'nick': self._get_symbol('topic')}

    def _msg_me(self, msg):
        """return data for /me"""
        return {'msg': msg.text,
                'nick': self._get_symbol('me')}

    def _msg_file(self, msg):
        """return data for file"""
        return {'msg': msg.text,
                'nick': self._get_symbol('file')}

    def _msg(self, msg):
        """return data for all other message types"""
        return {'msg': msg.text,
                'nick': msg.user.name}
<commit_msg>Small fixes for generating procedure, re-added emoji substitution<commit_after>"""
Reporters module.

There are several classes for specific format reporting, and also some of the
slack conversation/convention parsers.
"""
import os
import errno
import html.parser
import logging
import re

from slack_backup import objects as o
from slack_backup import utils
from slack_backup import emoji


class Reporter(object):
    """Base reporter class"""
    ext = ''
    symbols = {'plain': {'join': '->',
                         'leave': '<-',
                         'me': '*',
                         'file': '-',
                         'topic': '+',
                         'separator': '|'},
               'unicode': {'join': '⮊',
                           'leave': '⮈',
                           'me': '🟊',
                           'file': '📂',
                           'topic': '🟅',
                           'separator': '│'}}
    url_pat = re.compile(r'(?P<replace><http[^>]+>)')
    slackid_pat = re.compile(r'(?P<replace><@'
                             '(?P<slackid>U[A-Z,0-9]+)(\|[^>]+)?[^>]*>)')

    def __init__(self, args, query):
        self.out = args.output
        self.theme = args.theme
        self.q = query
        self.types = {"channel_join": self._msg_join,
                      "channel_leave": self._msg_leave,
                      "channel_topic": self._msg_topic,
                      "file_share": self._msg_file,
                      "me_message": self._msg_me}

        self.emoji = emoji.EMOJI.get(args.theme, {})

        self.channels = self._get_channels(args.channels)
        self.users = self.q(o.User).all()

    def generate(self):
        """Generate raport for each channel"""
        for channel in self.channels:
            messages = []
            log_path = self.get_log_path(channel.name)
            try:
                os.unlink(log_path)
            except IOError as err:
                if err.errno != errno.ENOENT:
                    raise
            for message in self.q(o.Message).\
                    filter(o.Message.channel == channel).\
                    order_by(o.Message.ts).all():
                messages.append(message)
            self.write_msg(messages, log_path)

    def get_log_path(self, name):
        """Return relative log file name """
        return os.path.join(self.out, name + self.ext)

    def write_msg(self, messages, log):
        """Write message to file"""
        with open(log, "a") as fobj:
            for message in messages:
                data = self._process_message(message)
                fobj.write(data['tpl'].format(**data))

    def _get_symbol(self, item):
        """Return appropriate item depending on the selected theme"""
        return self.symbols[self.theme][item]

    def _get_channels(self, selected_channels):
        """
        Retrieve channels from db and return those which names matched from
        selected_channels list
        """
        all_channels = self.q(o.Channel).all()
        if not selected_channels:
            return all_channels

        result = []
        for channel in all_channels:
            if channel.name in selected_channels:
                result.append(channel)
        return result

    def _process_message(self, msg):
        """
        Make changes to the text (replace slack ids, replace representation of
        urls, substitute images etc) and return dict with data suitable to
        display.
        """
        processor = self.types.get(msg.type, self._msg)
        data = processor(msg)
        data.update({'date': msg.datetime().strftime("%Y-%m-%d %H:%M:%S"),
                     'tpl': "{date} {nick} {msg}"})

        for emoticon in self.emoji:
            data['msg'] = data['msg'].replace(emoticon, self.emoji[emoticon])

        return data

    def _msg_join(self, msg):
        """return data for join"""
        return {'msg': msg.text,
                'nick': self._get_symbol('join')}

    def _msg_leave(self, msg):
        """return data for leave"""
        return {'msg': msg.text,
                'nick': self._get_symbol('leave')}

    def _msg_topic(self, msg):
        """return data for set topic"""
        return {'msg': msg.text,
                'nick': self._get_symbol('topic')}

    def _msg_me(self, msg):
        """return data for /me"""
        return {'msg': msg.user.name + ' ' + msg.text,
                'nick': self._get_symbol('me')}

    def _msg_file(self, msg):
        """return data for file"""
        return {'msg': msg.text,
                'nick': self._get_symbol('file')}

    def _msg(self, msg):
        """return data for all other message types"""
        return {'msg': msg.text,
                'nick': msg.user.name}

{'fraction_non_alphanumeric': 0.1343061028863564, 'fraction_numerical': 0.0004260304611779742, 'mean_word_length': 1.7384076990376203, 'pattern_counts': {'":': 10, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7026488', 'n_tokens_mistral': 2709, 'n_tokens_neox': 2590, 'n_words': 680}