<commit_before># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Union

from dino import environ
from dino import utils
from dino.config import ConfigKeys
from dino.utils.decorators import timeit

import logging
import traceback
import json
import sys
import emoji

__author__ = '<NAME> <<EMAIL>>'

logger = logging.getLogger(__name__)


class OnMessageHooks(object):
    @staticmethod
    def do_process(arg: tuple) -> None:
        def send(_data: dict, _room: str, _json: bool=True, _broadcast: bool=True) -> None:
            environ.env.emit('message', _data, json=_json, room=_room, broadcast=_broadcast)

        @timeit(logger, 'on_message_hooks_publish_activity')
        def publish_activity() -> None:
            user_name = activity.actor.display_name
            if utils.is_base64(user_name):
                user_name = utils.b64d(user_name)

            activity_json = utils.activity_for_message(user_id, user_name)
            environ.env.publish(activity_json, external=True)

        @timeit(logger, 'on_message_hooks_broadcast')
        def broadcast():
            room_id = activity.target.id
            if utils.user_is_invisible(user_id):
                data['actor']['attachments'] = utils.get_user_info_attachments_for(user_id)

            if activity.target.object_type == 'private':
                owners = environ.env.db.get_owners_room(activity.target.id)
                if owners is None or len(owners) == 0:
                    send(data, _room=room_id)
                else:
                    for owner in owners:
                        send(data, _room=owner)
            else:
                send(data, _room=room_id)

        @timeit(logger, 'on_message_hooks_store')
        def store(deleted=False) -> Union[str, None]:
            try:
                message_id = environ.env.storage.store_message(activity, deleted=deleted)
            except Exception as e:
                logger.error('could not store message %s because: %s' % (activity.id, str(e)))
                logger.error(str(data))
                logger.exception(traceback.format_exc())
                environ.env.capture_exception(sys.exc_info())
                return

            if not environ.env.config.get(ConfigKeys.DELIVERY_GUARANTEE, False) or \
                    activity.target.object_type != 'private':
                return

            owners = environ.env.db.get_owners_room(activity.target.id)
            environ.env.storage.mark_as_read({activity.id}, activity.actor.id, activity.target.id)
            if owners is None or len(owners) == 0:
                return

            for receiver_id in owners:
                if activity.actor.id == receiver_id:
                    continue
                environ.env.storage.mark_as_unacked(activity.id, receiver_id, activity.target.id)

            return message_id

        def check_spam():
            def text_without_emoji(text):
                return [character for character in text if character not in emoji.UNICODE_EMOJI]

            _is_spam = False
            _spam_id = None
            _message = None

            spam_enabled = environ.env.config.get(ConfigKeys.SPAM_CLASSIFIER, False)
            if not spam_enabled:
                return False, None

            try:
                _message = utils.b64d(activity.object.content)
                try:
                    json_body = json.loads(_message)
                    _message = json_body.get('text')
                except Exception:
                    pass  # ignore, use original
            except Exception as e:
                logger.error('could not decode message: {}'.format(str(e)))
                logger.exception(e)
                environ.env.capture_exception(sys.exc_info())
                return False, None

            if environ.env.service_config.ignore_emoji():
                try:
                    _message = text_without_emoji(_message)
                except Exception as e:
                    logger.error('could not check if text has emojis: {}'.format(str(e)))
                    logger.exception(e)
                    environ.env.capture_exception(sys.exc_info())

            try:
                _is_spam, _y_hats = environ.env.spam.is_spam(_message)
                if _is_spam and environ.env.service_config.should_save_spam():
                    _spam_id = environ.env.db.save_spam_prediction(activity, _message, _y_hats)
            except Exception as e:
                logger.error('could not predict spam: {}'.format(str(e)))
                logger.exception(e)
                environ.env.capture_exception(sys.exc_info())
                return False, None

            return _is_spam, _spam_id

        data, activity = arg
        user_id = activity.actor.id
        user_used_blacklisted_word, word_used_if_any = utils.used_blacklisted_word(activity)

        if user_used_blacklisted_word:
            blacklist_activity = utils.activity_for_blacklisted_word(activity, word_used_if_any)
            environ.env.publish(blacklist_activity, external=True)
            send(data, _room=user_id, _broadcast=False)

            admins_in_room = environ.env.db.get_admins_in_room(activity.target.id)<commit_msg>remove leading username reference and custom emojis<commit_after># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Union

from dino import environ
from dino import utils
from dino.config import ConfigKeys
from dino.utils.decorators import timeit

import logging
import traceback
import json
import sys
import emoji
import re

__author__ = '<NAME> <<EMAIL>>'

logger = logging.getLogger(__name__)


class OnMessageHooks(object):
    @staticmethod
    def do_process(arg: tuple) -> None:
        def send(_data: dict, _room: str, _json: bool=True, _broadcast: bool=True) -> None:
            environ.env.emit('message', _data, json=_json, room=_room, broadcast=_broadcast)

        @timeit(logger, 'on_message_hooks_publish_activity')
        def publish_activity() -> None:
            user_name = activity.actor.display_name
            if utils.is_base64(user_name):
                user_name = utils.b64d(user_name)

            activity_json = utils.activity_for_message(user_id, user_name)
            environ.env.publish(activity_json, external=True)

        @timeit(logger, 'on_message_hooks_broadcast')
        def broadcast():
            room_id = activity.target.id
            if utils.user_is_invisible(user_id):
                data['actor']['attachments'] = utils.get_user_info_attachments_for(user_id)

            if activity.target.object_type == 'private':
                owners = environ.env.db.get_owners_room(activity.target.id)
                if owners is None or len(owners) == 0:
                    send(data, _room=room_id)
                else:
                    for owner in owners:
                        send(data, _room=owner)
            else:
                send(data, _room=room_id)

        @timeit(logger, 'on_message_hooks_store')
        def store(deleted=False) -> Union[str, None]:
            try:
                message_id = environ.env.storage.store_message(activity, deleted=deleted)
            except Exception as e:
                logger.error('could not store message %s because: %s' % (activity.id, str(e)))
                logger.error(str(data))
                logger.exception(traceback.format_exc())
                environ.env.capture_exception(sys.exc_info())
                return

            if not environ.env.config.get(ConfigKeys.DELIVERY_GUARANTEE, False) or \
                    activity.target.object_type != 'private':
                return

            owners = environ.env.db.get_owners_room(activity.target.id)
            environ.env.storage.mark_as_read({activity.id}, activity.actor.id, activity.target.id)
            if owners is None or len(owners) == 0:
                return

            for receiver_id in owners:
                if activity.actor.id == receiver_id:
                    continue
                environ.env.storage.mark_as_unacked(activity.id, receiver_id, activity.target.id)

            return message_id

        def check_spam():
            def text_without_emoji(text):
                return ''.join([character for character in text if character not in emoji.UNICODE_EMOJI])

            def remove_custom_emojis(text):
                return re.sub(r':[a-z]*:', '', text)

            _is_spam = False
            _spam_id = None
            _message = None

            spam_enabled = environ.env.config.get(ConfigKeys.SPAM_CLASSIFIER, False)
            if not spam_enabled:
                return False, None

            try:
                _message = utils.b64d(activity.object.content)
                try:
                    json_body = json.loads(_message)
                    _message = json_body.get('text')
                except Exception:
                    pass  # ignore, use original
            except Exception as e:
                logger.error('could not decode message: {}'.format(str(e)))
                logger.exception(e)
                environ.env.capture_exception(sys.exc_info())
                return False, None

            if environ.env.service_config.ignore_emoji():
                try:
                    _message = text_without_emoji(_message)
                    _message = remove_custom_emojis(_message)
                except Exception as e:
                    logger.error('could not check if text has emojis: {}'.format(str(e)))
                    logger.exception(e)
                    environ.env.capture_exception(sys.exc_info())

            try:
                _message = _message.strip().lstrip('-')
                _is_spam, _y_hats = environ.env.spam.is_spam(_message)
                if _is_spam and environ.env.service_config.should_save_spam():
                    _spam_id = environ.env.db.save_spam_prediction(activity, _message, _y_hats)
            except Exception as e:
                logger.error('could not predict spam: {}'.format(str(e)))
                logger.exception(e)
                environ.env.capture_exception(sys.exc_info())
                return False, None

            return _is_spam, _spam_id

        data, activity = arg
        user_id = activity.actor.id
        user_used_blacklisted_word, word_used_if_any = utils.used_blacklisted_word(activity)

        if user_used_blacklisted_word:
            blacklist_activity = utils.activity_for_blacklisted_word(activity, word_used_if_any)
            environ.env.publish(blacklist_activity, external=True)
            send(data, _room=user_id, _broadcast=False)

            admins_in_room = environ.env.db.get_admins_in_room(activity.target.id)
{'fraction_non_alphanumeric': 0.08627584446524292, 'fraction_numerical': 0.002042031821662554, 'mean_word_length': 2.072138003136435, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1032025', 'n_tokens_mistral': 3301, 'n_tokens_neox': 3223, 'n_words': 762}