<commit_before># Name: slack_logger.py
# Author: pbzweihander
# Email: <EMAIL>
#
# Copyright (C) 2017 pbzweihander
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

import datetime
import elasticsearch_wrapper
import slack_wrapper
import settings
import json
import logging.handlers


class SlackLogger:
    slack = slack_wrapper.Slack(settings.SLACK_TOKEN, settings.BOT_NAME)
    logger = logging.getLogger('slack-logger')
    logger.setLevel(logging.INFO)
    formatter = logging.Formatter('%(message)s')
    file_handler = logging.handlers.TimedRotatingFileHandler(
        './logs/slack-log', when='midnight', interval=1, encoding="UTF-8")
    file_handler.suffix = "%Y-%m-%d"
    stream_handler = logging.StreamHandler()

    last_search = ()

    def __init__(self):
        self.logger.addHandler(self.file_handler)
        self.logger.addHandler(self.stream_handler)

    @staticmethod
    def parse_search_filter(ts: list) -> list:
        if len(ts) == 1:
            return ts
        t = ts.pop()
        if ':' in t:
            return SlackLogger.parse_search_filter(ts) + [t]
        else:
            ts[-1] += ' ' + t
            return SlackLogger.parse_search_filter(ts)

    @staticmethod
    def log_help() -> list:
        return [{'text': "Usage:\n"
                         "!logsearch <key1>:<value1> [<key2>:<value2> ...]\n"
                         "!logmore [<size>]\n"
                         "!loghelp\n"
                         "(key: channel, user, text, time)"}]

    def log_more(self, size=10) -> list:
        if not self.last_search:
            return [{'text': "마지막 검색 결과가 없습니다."}]
        return self.log_search(self.last_search[0], size, self.last_search[1])

    def log_search(self, filters: list, size=10, fr=None) -> list:
        if len(filters) > 1:
            res = elasticsearch_wrapper.es_filter_search(
                settings.ES_INDEX, settings.ES_TYPE, filters, size, fr, [{'time': {'order': 'desc'}}])
        else:
            res = elasticsearch_wrapper.es_single_search(
                settings.ES_INDEX, settings.ES_TYPE,{filters[0][0]: filters[0][1]},
                size, fr, [{'time': {'order': 'desc'}}])
        outdict = dict()
        outdict['pretext'] = "%s개가 검색됨" % len(res)
        outdict['title'] = ', '.join(["%s: %s" % (f[0], f[1]) for f in filters])
        if res:
            outdict['text'] = '\n'.join(["%s %s@%s: %s" %
                                         (doc[1]['time'], doc[1]['user'], doc[1]['channel'], doc[1]['text'])
                                         for doc in res][::-1])
            self.last_search = (filters, res[-1][0])
        else:
            outdict['text'] = "검색 결과가 없습니다."
        return [outdict]

    def handle_command(self, text: str) -> list:
        if text.startswith("!"):
            if text.split()[0] == "!logsearch":
                if len(text.split()) == 1:
                    return self.log_help()
                filters = [tuple(map(str.strip, f.split(':')))
                           for f in self.parse_search_filter(text.split()[1:])]
                if not filters:
                    return self.log_help()
                return self.log_search(filters)
            elif text.split()[0] == "!logmore":
                args = text.split()
                if len(args) > 1:
                    return self.log_more(int(args[1]))
                else:
                    return self.log_more()
            elif text.split()[0] == "!loghelp":
                return self.log_help()
        return []

    def handle_message(self, d: dict):
        channame = self.slack.channels.get(d['channel'])
        if not channame:
            self.slack.refresh_channels()
            channame = self.slack.channels.get(d['channel']) or "?"
        username = self.slack.users.get(d['user'])
        if not username:
            self.slack.refresh_users()
            username = self.slack.users.get(d['user']) or "?"
        date_ts = datetime.datetime.fromtimestamp(float(d['ts']))
        self.log_to_json(channame, username, d['text'], date_ts)
        res = self.handle_command(d['text'])
        if res:
            self.slack.post_formatted_message(d['channel'], res)
        return ""
<commit_msg>찰진검색에 맞게 파서를 재구현<commit_after># Name: slack_logger.py
# Author: pbzweihander
# Email: <EMAIL>
#
# Copyright (C) 2017 pbzweihander
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

import datetime
import elasticsearch_wrapper
import slack_wrapper
import settings
import json
import logging.handlers


class SlackLogger:
    slack = slack_wrapper.Slack(settings.SLACK_TOKEN, settings.BOT_NAME)
    logger = logging.getLogger('slack-logger')
    logger.setLevel(logging.INFO)
    formatter = logging.Formatter('%(message)s')
    file_handler = logging.handlers.TimedRotatingFileHandler(
        './logs/slack-log', when='midnight', interval=1, encoding="UTF-8")
    file_handler.suffix = "%Y-%m-%d"
    stream_handler = logging.StreamHandler()

    last_search = ()

    def __init__(self):
        self.logger.addHandler(self.file_handler)
        self.logger.addHandler(self.stream_handler)

    @staticmethod
    def parse_search_filter(ts: list, key: str) -> list:
        if not ts:
            return []
        if ':' in ts[0]:
            key = ts[0].split(':')[0]
        return [(key, ts[0].split(':')[1])] + SlackLogger.parse_search_filter(ts[1:], key)

    @staticmethod
    def log_help() -> list:
        return [{'text': "Usage:\n"
                         "!logsearch <key1>:<value1> [<key2>:<value2> [...]]\n"
                         "!logmore [<size>]\n"
                         "!loghelp\n"
                         "(key: channel, user, text, time)"}]

    def log_more(self, size=10) -> list:
        if not self.last_search:
            return [{'text': "마지막 검색 결과가 없습니다."}]
        return self.log_search(self.last_search[0], size, self.last_search[1])

    def log_search(self, filters: list, size=10, fr=None) -> list:
        if len(filters) > 1:
            res = elasticsearch_wrapper.es_filter_search(
                settings.ES_INDEX, settings.ES_TYPE, filters, size, fr, [{'time': {'order': 'desc'}}])
        else:
            res = elasticsearch_wrapper.es_single_search(
                settings.ES_INDEX, settings.ES_TYPE,{filters[0][0]: filters[0][1]},
                size, fr, [{'time': {'order': 'desc'}}])
        outdict = dict()
        outdict['pretext'] = "%s개가 검색됨" % len(res)
        outdict['title'] = ', '.join(["%s: %s" % (f[0], f[1]) for f in filters])
        if res:
            outdict['text'] = '\n'.join(["%s %s@%s: %s" %
                                         (doc[1]['time'], doc[1]['user'], doc[1]['channel'], doc[1]['text'])
                                         for doc in res][::-1])
            self.last_search = (filters, res[-1][0])
        else:
            outdict['text'] = "검색 결과가 없습니다."
        return [outdict]

    def handle_command(self, text: str) -> list:
        if text.startswith("!"):
            if text.split()[0] == "!logsearch":
                if len(text.split()) == 1:
                    return self.log_help()
                filters = self.parse_search_filter(text.split()[1:], 'text')
                if not filters:
                    return self.log_help()
                return self.log_search(filters)
            elif text.split()[0] == "!logmore":
                args = text.split()
                if len(args) > 1:
                    return self.log_more(int(args[1]))
                else:
                    return self.log_more()
            elif text.split()[0] == "!loghelp":
                return self.log_help()
        return []

    def handle_message(self, d: dict):
        channame = self.slack.channels.get(d['channel'])
        if not channame:
            self.slack.refresh_channels()
            channame = self.slack.channels.get(d['channel']) or "?"
        username = self.slack.users.get(d['user'])
        if not username:
            self.slack.refresh_users()
            username = self.slack.users.get(d['user']) or "?"
        date_ts = datetime.datetime.fromtimestamp(float(d['ts']))
        self.log_to_json(channame, username, d['text'], date_ts)
        res = self.handle_command(d['text'])
        if res:
            self.slack.post_formatted_message(d['channel'], res)
        return ""

{'fraction_non_alphanumeric': 0.14076467420570812, 'fraction_numerical': 0.009046849757673668, 'mean_word_length': 2.162806539509537, 'pattern_counts': {'":': 6, '<': 15, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5675210', 'n_tokens_mistral': 2892, 'n_tokens_neox': 2842, 'n_words': 679}