<commit_before>from __future__ import print_function
import sys
import os

import json
import logging
import re
import urllib

from firebase import firebase

logger = logging.getLogger(__name__)


def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)

class RtmEventHandler(object):
    def __init__(self, slack_clients, msg_writer):
        self.clients = slack_clients
        self.msg_writer = msg_writer
        self.firebase = firebase.FirebaseApplication(os.getenv("FIREBASE_URL"), None)
        self.firebase.authentication = firebase.FirebaseAuthentication(os.getenv("FIREBASE_SECRET"), os.getenv("FIREBASE_EMAIL"))

    def handle(self, event):

        if 'type' in event:
            self._handle_by_type(event['type'], event)

    def _handle_by_type(self, event_type, event):
        # See https://api.slack.com/rtm for a full list of events
        if event_type == 'error':
            # error
            self.msg_writer.write_error(event['channel'], json.dumps(event))
        elif event_type == 'message':
            # message was sent to channel
            self._handle_message(event)
        elif event_type == 'channel_joined':
            # you joined a channel
            self.msg_writer.write_help_message(event['channel'])
        elif event_type == 'group_joined':
            # you joined a private group
            self.msg_writer.write_help_message(event['channel'])
        else:
            pass

    def _save(self, key, value):
        if self._get(key):
            return False
        self.firebase.post('/glossary/' + urllib.quote_plus(key.lower().encode('utf-8')), value)
        return True

    def _get(self, key):
        result = self.firebase.get('/glossary', urllib.quote_plus(key.lower().encode('utf-8')))
        if result:
            return result.itervalues().next()
        return False

    def _clean_links(self, message):
        return re.sub("<([^@\#\!].*?)(\|.*?)?>", r'\1', message)

    def _is_hidden_message_event(self, event):
        return 'hidden' in event and event['hidden'] == True

    def _handle_message(self, event):
        # Filter out messages from the bot itself
        if not self._is_hidden_message_event(event) and not self.clients.is_message_from_me(event['user']):

            msg_txt = event['text']

            if self.clients.is_bot_mention(msg_txt):

                add_definition_regexp = re.search("^<@{}>[\s:]+(.+)\s*=\s*(.+)".format(re.escape(self.clients.bot_user_id())), msg_txt, re.MULTILINE|re.DOTALL)
                get_definition_regexp = re.search("^<@{}>[\s:]+(.+)".format(re.escape(self.clients.bot_user_id())), msg_txt, re.MULTILINE|re.DOTALL)
                <commit_msg>Update event_handler.py<commit_after>from __future__ import print_function
import sys
import os

import json
import logging
import re
import urllib

from firebase import firebase

logger = logging.getLogger(__name__)


def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)

class RtmEventHandler(object):
    def __init__(self, slack_clients, msg_writer):
        self.clients = slack_clients
        self.msg_writer = msg_writer
        self.firebase = firebase.FirebaseApplication(os.getenv("FIREBASE_URL"), None)
        self.firebase.authentication = firebase.FirebaseAuthentication(os.getenv("FIREBASE_SECRET"), os.getenv("FIREBASE_EMAIL"))

    def handle(self, event):
        if 'type' in event:
            self._handle_by_type(event['type'], event)

    def _handle_by_type(self, event_type, event):
        # See https://api.slack.com/rtm for a full list of events
        if event_type == 'error':
            # error
            self.msg_writer.write_error(event['channel'], json.dumps(event))
        elif event_type == 'message':
            # message was sent to channel
            self._handle_message(event)
        elif event_type == 'channel_joined':
            # you joined a channel
            self.msg_writer.write_help_message(event['channel'])
        elif event_type == 'group_joined':
            # you joined a private group
            self.msg_writer.write_help_message(event['channel'])
        else:
            pass

    def _save(self, key, value):
        if self._get(key):
            return False
        self.firebase.post('/glossary/' + urllib.quote_plus(self._clean_dots(key.lower()).encode('utf-8')), value)
        return True

    def _get(self, key):
        result = self.firebase.get('/glossary', urllib.quote_plus(self._clean_dots(key.lower()).encode('utf-8')))
        if result:
            return result.itervalues().next()
        return False

    def _clean_links(self, message):
        return re.sub("<([^@\#\!].*?)(\|.*?)?>", r'\1', message)

    def _clean_dots(self, message):
        return re.sub('\.', '', message)

    def _is_hidden_message_event(self, event):
        return 'hidden' in event and event['hidden'] == True

    def _handle_message(self, event):
        # Filter out messages from the bot itself
        if not self._is_hidden_message_event(event) and not self.clients.is_message_from_me(event['user']):

            msg_txt = event['text']

            if self.clients.is_bot_mention(msg_txt):

                add_definition_regexp = re.search("^<@{}>[\s:]+(.+)\s*=\s*(.+)".format(re.escape(self.clients.bot_user_id())), msg_txt, re.MULTILINE|re.DOTALL)
                get_definition_regexp = re.search("^<@{}>[\s:]+(.+)".format(re.escape(self.clients.bot_user_id())), msg_txt, re.MULTILINE|re.DOTALL)
                
{'fraction_non_alphanumeric': 0.13656946205397572, 'fraction_numerical': 0.0010867596449918493, 'mean_word_length': 2.84808362369338, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '726823', 'n_tokens_mistral': 1754, 'n_tokens_neox': 1711, 'n_words': 345}