<commit_before>import os
import re
import time
import codecs
import hmac
import json
import hashlib
import datetime
import functools
import itertools
import sqlite3
from contextlib import closing

import pytz
import requests
import flask
from flask import Flask, request, redirect, session, url_for, render_template, jsonify

app = Flask(__name__)
app.config.from_envvar('LOGVIEWER_SETTINGS')

LINE_PATTERN = re.compile('^.*?\[(?P<timestamp>.+?)(?: #.*?)?\].*? (?P<dir><<<|>>>) (?P<data>.+)$')
PRIVMSG_PATTERN = re.compile('^(?::(?P<nick>.+?)!.+? )?PRIVMSG #.+? :(?P<text>.+)$')
PROXY_MSG_PATTERN = re.compile('<(?P<nick>.+?)> (?P<text>.*)')

def localize(t):
    return pytz.timezone(app.config['LOG_TIMEZONE']).localize(t)

def parse_log(fp, start=None):
    def extract_time(match):
        return localize(datetime.datetime.strptime(match.group('timestamp').split('.')[0], '%Y-%m-%dT%H:%M:%S'))

    no = 0
    for line in fp:
        m = LINE_PATTERN.match(line)
        if not m: continue

        if 'PRIVMSG' in m.group('data'):
            no += 1

            if start and no < start: continue
            pm = PRIVMSG_PATTERN.match(m.group('data'))
            if not pm: continue

            is_bot = not pm.group('nick')
            pmm = PROXY_MSG_PATTERN.match(pm.group('text'))
            if is_bot and pmm:
                nick = pmm.group('nick')
                text = pmm.group('text')
                is_bot = False
            else:
                nick = pm.group('nick')
                text = pm.group('text')

            yield dict(type='privmsg', no=no,
                time=extract_time(m),
                nick=nick, text=text, is_bot=is_bot)
        elif m.group('dir') == '>>>' and start is None \
            and m.group('data').startswith('JOIN #langdev'):
            yield dict(type='join', time=extract_time(m), is_bot=True, no=-1)

def filter_recent(iterable, minutes):
    now = localize(datetime.datetime.now())
    messages = list(iterable)
    start = 0
    for i in xrange(len(messages)):
        if (now - messages[i]['time']).seconds <= 60 * minutes:
            start = i
            break

    count = len(messages)
    if count - start < 50:
        start = count - 50

    if start == 0:
        return messages, False
    else:
        return messages[start:], True

def group_messages(messages, thres):
    it = iter(messages)
    prev_time = next(it)['time']
    group = []
    for msg in it:
        if (msg['time'] - prev_time).seconds > thres:
            yield group
            group = []
        group.append(msg)
        prev_time = msg['time']
    if group:
        yield group

import sphinxapi

def expand_synonyms(query):
    d = {}
    with codecs.open(app.config['SYNONYM_PATH'], encoding='utf-8') as fp:
        for line in fp:
            words = line.rstrip().split(' ')
            words.sort(key=len, reverse=True)
            for word in words:
                d[word] = words

    terms = query.split()
    expanded_terms = []
    for term in terms:
        if term in d:
            expanded_terms.append('(' + ')|('.join(d[term]) + ')')
        else:
            expanded_terms.append('(' + term + ')')
    return ' '.join(expanded_terms)

def sphinx_search(query, offset, count):
    client = sphinxapi.SphinxClient()
    client.SetServer('localhost', 9312)
    client.SetWeights([100, 1])
    client.SetSortMode(sphinxapi.SPH_SORT_EXTENDED, '@id DESC')
    client.SetLimits(offset, count, 100000)
    client.SetRankingMode(sphinxapi.SPH_RANK_PROXIMITY_BM25)
    
    client.SetMatchMode(sphinxapi.SPH_MATCH_BOOLEAN)
    result = client.Query(query, '*')
    if result:
        if 'matches' in result:
            messages = {}
            for msg in result['matches']:
                attrs = msg['attrs']
                t = time.localtime(attrs['time'])
                key = time.strftime('%Y%m%d', t)
                if key not in messages:
                    messages[key] = []
                messages[key].append(dict(type='privmsg', no=attrs['no'], time=localize(datetime.datetime(*t[:7])), nick=attrs['nick'].decode('utf-8'), text=attrs['content'].decode('utf-8'), is_bot=attrs['bot']))
            return dict(total=result['total'], messages=((Log(datetime.datetime.strptime(k, "%Y%m%d").date()), v) for k, v in sorted(messages.iteritems(), key=lambda (k, v): k, reverse=True)))

class Log(object):
    def __init__(self, date):
        self.date = date

    @property
    def is_today(self):
        return self.date == datetime.date.today()

    @property
    def path(self):
        path = os.path.join(app.config['LOG_DIR'], 'langdev.log')
        if not self.is_today:
            return path + '.' + self.date.strftime('%Y%m%d')
        else:
            return path

    @property
    def url(self):
        return url_for('log', date=self.date)

    def get_messages(self, start=None):
        with codecs.open(self.path, encoding='utf-8', errors='replace') as fp:
            for msg in parse_log(fp, start):
                yield msg

    @staticmethod
    def today():
        return Log(datetime.date.today())

    @property
    def yesterday(self):
        return Log(self.date - datetime.timedelta(days=1))

def langdev_sso_call(user_id, user_pass):
    def hmac_sha1(value):
        hash = hmac.new(app.config['LANGDEV_SECRET_KEY'], value, hashlib.sha1)
        return hash.hexdigest()
    def hmac_pass(u_pass):
        return hmac_<PASSWORD>1(hashlib.md5(u_pass).hexdigest())

    auth_url = 'http://langdev.org/apps/%s/sso/%s' % (app.config['LANGDEV_APP_KEY'], user_id)
    auth_data = {'password': <PASSWORD>(<PASSWORD>) }
    result = requests.post(auth_url, data=auth_data, headers={'Accept': 'application/json'})

    if result.status_code == requests.codes.ok:
        return json.loads(result.content)
    else:
        return False

def hashed(value, limit=0):
    hashed_value = hash(value)
    if limit:
        return hashed_value % limit
    else:
        return hashed_value
app.jinja_env.filters['hash'] = hashed

def login_required(f):
    @functools.wraps(f)
    def _wrapped(*args, **kwargs):
        if 'username' not in session:
        	return redirect(url_for('login', next=request.url))
        return f(*args, **kwargs)
    return _wrapped

@app.route('/login', methods=['GET', 'POST'])<commit_msg>colorize based on basenick<commit_after>import os
import re
import time
import codecs
import hmac
import json
import hashlib
import datetime
import functools
import itertools
import sqlite3
from contextlib import closing

import pytz
import requests
import flask
from flask import Flask, request, redirect, session, url_for, render_template, jsonify

app = Flask(__name__)
app.config.from_envvar('LOGVIEWER_SETTINGS')

LINE_PATTERN = re.compile('^.*?\[(?P<timestamp>.+?)(?: #.*?)?\].*? (?P<dir><<<|>>>) (?P<data>.+)$')
PRIVMSG_PATTERN = re.compile('^(?::(?P<nick>.+?)!.+? )?PRIVMSG #.+? :(?P<text>.+)$')
PROXY_MSG_PATTERN = re.compile('<(?P<nick>.+?)> (?P<text>.*)')

def localize(t):
    return pytz.timezone(app.config['LOG_TIMEZONE']).localize(t)

def parse_log(fp, start=None):
    def extract_time(match):
        return localize(datetime.datetime.strptime(match.group('timestamp').split('.')[0], '%Y-%m-%dT%H:%M:%S'))

    no = 0
    for line in fp:
        m = LINE_PATTERN.match(line)
        if not m: continue

        if 'PRIVMSG' in m.group('data'):
            no += 1

            if start and no < start: continue
            pm = PRIVMSG_PATTERN.match(m.group('data'))
            if not pm: continue

            is_bot = not pm.group('nick')
            pmm = PROXY_MSG_PATTERN.match(pm.group('text'))
            if is_bot and pmm:
                nick = pmm.group('nick')
                text = pmm.group('text')
                is_bot = False
            else:
                nick = pm.group('nick')
                text = pm.group('text')

            yield dict(type='privmsg', no=no,
                time=extract_time(m),
                nick=nick, text=text, is_bot=is_bot)
        elif m.group('dir') == '>>>' and start is None \
            and m.group('data').startswith('JOIN #langdev'):
            yield dict(type='join', time=extract_time(m), is_bot=True, no=-1)

def filter_recent(iterable, minutes):
    now = localize(datetime.datetime.now())
    messages = list(iterable)
    start = 0
    for i in xrange(len(messages)):
        if (now - messages[i]['time']).seconds <= 60 * minutes:
            start = i
            break

    count = len(messages)
    if count - start < 50:
        start = count - 50

    if start == 0:
        return messages, False
    else:
        return messages[start:], True

def group_messages(messages, thres):
    it = iter(messages)
    prev_time = next(it)['time']
    group = []
    for msg in it:
        if (msg['time'] - prev_time).seconds > thres:
            yield group
            group = []
        group.append(msg)
        prev_time = msg['time']
    if group:
        yield group

import sphinxapi

def expand_synonyms(query):
    d = {}
    with codecs.open(app.config['SYNONYM_PATH'], encoding='utf-8') as fp:
        for line in fp:
            words = line.rstrip().split(' ')
            words.sort(key=len, reverse=True)
            for word in words:
                d[word] = words

    terms = query.split()
    expanded_terms = []
    for term in terms:
        if term in d:
            expanded_terms.append('(' + ')|('.join(d[term]) + ')')
        else:
            expanded_terms.append('(' + term + ')')
    return ' '.join(expanded_terms)

def sphinx_search(query, offset, count):
    client = sphinxapi.SphinxClient()
    client.SetServer('localhost', 9312)
    client.SetWeights([100, 1])
    client.SetSortMode(sphinxapi.SPH_SORT_EXTENDED, '@id DESC')
    client.SetLimits(offset, count, 100000)
    client.SetRankingMode(sphinxapi.SPH_RANK_PROXIMITY_BM25)
    
    client.SetMatchMode(sphinxapi.SPH_MATCH_BOOLEAN)
    result = client.Query(query, '*')
    if result:
        if 'matches' in result:
            messages = {}
            for msg in result['matches']:
                attrs = msg['attrs']
                t = time.localtime(attrs['time'])
                key = time.strftime('%Y%m%d', t)
                if key not in messages:
                    messages[key] = []
                messages[key].append(dict(type='privmsg', no=attrs['no'], time=localize(datetime.datetime(*t[:7])), nick=attrs['nick'].decode('utf-8'), text=attrs['content'].decode('utf-8'), is_bot=attrs['bot']))
            return dict(total=result['total'], messages=((Log(datetime.datetime.strptime(k, "%Y%m%d").date()), v) for k, v in sorted(messages.iteritems(), key=lambda (k, v): k, reverse=True)))

class Log(object):
    def __init__(self, date):
        self.date = date

    @property
    def is_today(self):
        return self.date == datetime.date.today()

    @property
    def path(self):
        path = os.path.join(app.config['LOG_DIR'], 'langdev.log')
        if not self.is_today:
            return path + '.' + self.date.strftime('%Y%m%d')
        else:
            return path

    @property
    def url(self):
        return url_for('log', date=self.date)

    def get_messages(self, start=None):
        with codecs.open(self.path, encoding='utf-8', errors='replace') as fp:
            for msg in parse_log(fp, start):
                yield msg

    @staticmethod
    def today():
        return Log(datetime.date.today())

    @property
    def yesterday(self):
        return Log(self.date - datetime.timedelta(days=1))

def langdev_sso_call(user_id, user_pass):
    def hmac_sha1(value):
        hash = hmac.new(app.config['LANGDEV_SECRET_KEY'], value, hashlib.sha1)
        return hash.hexdigest()
    def hmac_pass(u_pass):
        return hmac_sha1(hashlib.md5(u_pass).hexdigest())

    auth_url = 'http://langdev.org/apps/%s/sso/%s' % (app.config['LANGDEV_APP_KEY'], user_id)
    auth_data = {'password': <PASSWORD>) }
    result = requests.post(auth_url, data=auth_data, headers={'Accept': 'application/json'})

    if result.status_code == requests.codes.ok:
        return json.loads(result.content)
    else:
        return False

def hashed(value, limit=0):
    m = re.search(r'^[\^\|_]*([^\^\|_]*).*$', value)
    if not m is None:
        value = m.group(1)
    hashed_value = hash(value)
    if limit:
        return hashed_value % limit
    else:
        return hashed_value
app.jinja_env.filters['hash'] = hashed

def login_required(f):
    @functools.wraps(f)
    def _wrapped(*args, **kwargs):
        if 'username' not in session:
        	return redirect(url_for('login', next=request.url))
        return f(*args, **kwargs)
    return _wrapped

@app.route('/login', methods=['GET', 'POST'])
{'fraction_non_alphanumeric': 0.15230178025252922, 'fraction_numerical': 0.006352442945651322, 'mean_word_length': 2.557043235704324, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3899637', 'n_tokens_mistral': 4300, 'n_tokens_neox': 4139, 'n_words': 902}