<commit_before>#!/usr/bin/python

# Read irc logs from our private channel and post them to our wiki

import glob
import json
import os
import re
import sys
import textwrap

from simplemediawiki import MediaWiki


with open(os.path.expanduser('~/.mediawiki'), 'r') as f:
    conf = json.loads(f.read())['ircbot']

wiki = MediaWiki(conf['url'])

day_re = re.compile('^--- Day changed (.*) (.*) ([0-9]+) (20[0-9]+)$')
human_re = re.compile('.*<([^>]+)>.*')

days = {}
days_order = []


def make_wiki_login_call(packet):
    packet.update({'lgname': conf['username'],
                   'lgpassword': conf['password']})
    return wiki.call(packet)

def post_page(title, text, day_tuple):
    page_token = wiki.call({'action': 'query',
                            'prop': 'info',
                            'titles': title,
                            'intoken': 'edit'})
    pages = page_token['query']['pages']
    page_id = pages.keys()[0]

    response = wiki.call({'action': 'edit',
                          'minor': True,
                          'bot': True,
                          'title': title,
                          'text': json.dumps(text).replace('\\n', '\n')[1:-1],
                          'token': pages[page_id]['edittoken']})
    if not 'nochange' in response['edit']:
        print 'Modified %s' % title

    if day_tuple:
        key = '%s %s' %(day_tuple[1], day_tuple[3])
        days.setdefault(key, [])
        days[key].append('[[%s|%s]]' %(title, day_tuple[2]))

        if not key in days_order:
            days_order.append(key)


if __name__ == '__main__':
    login = make_wiki_login_call({'action': 'login'})
    token = make_wiki_login_call({'action': 'login',
                                  'lgtoken': login['login']['token']})

    day = None
    content = []

    filenames = sorted(glob.glob(conf['logpath']))
    ordered_filenames = filenames[1:]
    ordered_filenames.append(filenames[0])
    for filename in ordered_filenames:
        print filename

        with open(filename, 'r') as f:
            l = f.readline()
            while l:
                if l.startswith('--- Day'):
                    m = day_re.match(l)
                    if m:
                        print 'Day is %s' % day
                        if content:
                            post_page('rcbau irc log for %s' % ' '.join(day),
                                      ''.join(content), day)
                            content = []
                        day = [m.group(1), m.group(2), m.group(3), m.group(4)]
                elif day:
                    lines = textwrap.wrap(l.rstrip(), 120)

                    m = human_re.match(l)
                    if m and len(m.group(1)) > 1:
                        content.append(' \'\'\'%s\'\'\'\n'
                                       % '\n \'\'\'      '.join(lines))
                    elif l[7] == '*':
                        content.append(' \'\'%s\'\'\n'
                                       % '\n \'\'      '.join(lines))
                    else:
                        content.append(' %s\n' % '\n       '.join(lines))

                l = f.readline()

    if day and content:
        post_page('rcbau irc log for %s' % ' '.join(day),
                  ''.join(content), day)

    if days:
        content = ''
        for day in reversed(days_order):
            content += ' %s: ' % day
            content += ' '.join(days[day])
            content += '\n'

        post_page('rcbau irc log index', content, None)<commit_msg>Handle a day appearing more than once in a row.<commit_after>#!/usr/bin/python

# Read irc logs from our private channel and post them to our wiki

import glob
import json
import os
import re
import sys
import textwrap

from simplemediawiki import MediaWiki


with open(os.path.expanduser('~/.mediawiki'), 'r') as f:
    conf = json.loads(f.read())['ircbot']

wiki = MediaWiki(conf['url'])

day_re = re.compile('^--- Day changed (.*) (.*) ([0-9]+) (20[0-9]+)$')
human_re = re.compile('.*<([^>]+)>.*')

days = {}
days_order = []


def make_wiki_login_call(packet):
    packet.update({'lgname': conf['username'],
                   'lgpassword': conf['password']})
    return wiki.call(packet)

def post_page(title, text, day_tuple):
    page_token = wiki.call({'action': 'query',
                            'prop': 'info',
                            'titles': title,
                            'intoken': 'edit'})
    pages = page_token['query']['pages']
    page_id = pages.keys()[0]

    response = wiki.call({'action': 'edit',
                          'minor': True,
                          'bot': True,
                          'title': title,
                          'text': json.dumps(text).replace('\\n', '\n')[1:-1],
                          'token': pages[page_id]['edittoken']})
    if not 'nochange' in response['edit']:
        print 'Modified %s' % title

    if day_tuple:
        key = '%s %s' %(day_tuple[1], day_tuple[3])
        days.setdefault(key, [])
        days[key].append('[[%s|%s]]' %(title, day_tuple[2]))

        if not key in days_order:
            days_order.append(key)


if __name__ == '__main__':
    login = make_wiki_login_call({'action': 'login'})
    token = make_wiki_login_call({'action': 'login',
                                  'lgtoken': login['login']['token']})

    day = None
    content = []

    filenames = sorted(glob.glob(conf['logpath']))
    ordered_filenames = filenames[1:]
    ordered_filenames.append(filenames[0])
    for filename in ordered_filenames:
        print filename

        with open(filename, 'r') as f:
            l = f.readline()
            while l:
                if l.startswith('--- Day'):
                    m = day_re.match(l)
                    if m:
                        new_day = [m.group(1), m.group(2), m.group(3),
                                   m.group(4)]

                        print 'Day %s to %s' %(day, new_day)
                        if content and day != new_day:
                            post_page('rcbau irc log for %s' % ' '.join(day),
                                      ''.join(content), day)
                            content = []
                        day = new_day
                elif day:
                    lines = textwrap.wrap(l.rstrip(), 120)

                    m = human_re.match(l)
                    if m and len(m.group(1)) > 1:
                        content.append(' \'\'\'%s\'\'\'\n'
                                       % '\n \'\'\'      '.join(lines))
                    elif l[7] == '*':
                        content.append(' \'\'%s\'\'\n'
                                       % '\n \'\'      '.join(lines))
                    else:
                        content.append(' %s\n' % '\n       '.join(lines))

                l = f.readline()

    if day and content:
        post_page('rcbau irc log for %s' % ' '.join(day),
                  ''.join(content), day)

    if days:
        content = ''
        for day in reversed(days_order):
            content += ' %s: ' % day
            content += ' '.join(days[day])
            content += '\n'

        post_page('rcbau irc log index', content, None)
{'fraction_non_alphanumeric': 0.1592551417454141, 'fraction_numerical': 0.006670372429127293, 'mean_word_length': 1.4305977710233029, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5459024', 'n_tokens_mistral': 2225, 'n_tokens_neox': 2114, 'n_words': 489}