<commit_before>import json
import re
import time

translate_map = {ord(c): None for c in map(chr, list(range(256))) if not c.isalnum()}


class SlackEvent(object):
    """
    Event received from the Slack RTM API
    params:
     - event_obj(json object)
    attributes:
     - type(type): Slack event type
     - ts(float): UTC event timestamp
    """

    def __init__(self, event_obj):
        self.event = event_obj
        self.mentions = []

        if 'type' in self.event:
            self.type = self.event['type']

        if 'ts' in self.event:
            self.ts = self.event['ts']
        else:
            self.ts = int(time.time())

        if 'text' in self.event:
            self.mentions = self._get_mentions(self.event['text'])

    @property
    def json(self):
        return json.dumps(self.event)

    def _get_mentions(self, text):
        mentions = re.findall('<@\w+>', text)
        if mentions:
            return [str(m).translate(translate_map) for m in mentions]
        else:
            return []


class SlackMsg(object):
    """
    Slack default formatted message capable of being sent via the RTM API
    params:
     - text(str)
     - channel(str)
    attributes:
     - type: Slack event type
     - ts: UTC time event was received 
    """

    def __init__(self, id, channel, text):
        self.sent = False
        self.payload = {'id': id,
                        'type': 'message',
                        'text': text,
                        'channel': channel}
        self.json = json.dumps(self.payload)
<commit_msg>fix str/unicode bug in _get_mentions when using python2<commit_after>import re
import sys
import json
import time

translate_map = { ord(c): None for c in map(chr, list(range(256))) if not c.isalnum() }


class SlackEvent(object):
    """
    Event received from the Slack RTM API
    params:
     - event_obj(json object)
    attributes:
     - type(type): Slack event type
     - ts(float): UTC event timestamp
    """

    def __init__(self, event_obj):
        self.event = event_obj
        self.mentions = []

        if 'type' in self.event:
            self.type = self.event['type']

        if 'ts' in self.event:
            self.ts = self.event['ts']
        else:
            self.ts = int(time.time())

        if 'text' in self.event:
            self.mentions = self._get_mentions(self.event['text'])

    @property
    def json(self):
        return json.dumps(self.event)

    def _get_mentions(self, text):
        mentions = re.findall('<@\w+>', text)

        if mentions and sys.version_info.major == 2:
            return [ unicode(m).translate(translate_map) for m in mentions ]

        if mentions and sys.version_info.major == 3:
            return [ str(m).translate(translate_map) for m in mentions ]

        return []


class SlackMsg(object):
    """
    Slack default formatted message capable of being sent via the RTM API
    params:
     - text(str)
     - channel(str)
    attributes:
     - type: Slack event type
     - ts: UTC time event was received 
    """

    def __init__(self, id, channel, text):
        self.sent = False
        self.payload = {'id': id,
                        'type': 'message',
                        'text': text,
                        'channel': channel}
        self.json = json.dumps(self.payload)

{'fraction_non_alphanumeric': 0.11602870813397129, 'fraction_numerical': 0.0026913875598086126, 'mean_word_length': 1.9445422535211268, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7620854', 'n_tokens_mistral': 1030, 'n_tokens_neox': 987, 'n_words': 286}