<commit_before># -*- coding: utf-8 -*-
import re
import db
import config
import requests
from time import sleep
from parsel import Selector
from datetime import datetime

_xpath_topics = '//*[@id="ipbcontent"]/div/ol/li/a'
_xpath_messages = '//*[@id="print"]/div[@class="printpost"]'
_url_lofi_page = 'http://diesel.elcat.kg/lofiversion/index.php?f{0}-{1}.html'
_url_print_page = 'http://diesel.elcat.kg/index.php?act=Print&client=printer&f=1&t={0}'


def _grab_topics(fid):
    topics = []
    for page in range(config.forum_pages_depth):
        print str(fid) + ':' + str(page + 1)
        # lite version topics list contains 150 topics per page
        text = requests.get(_url_lofi_page.format(fid, 150 * page)).text
        selector = Selector(text=text)
        t = selector.xpath(_xpath_topics).re(r'\?t([\d]+).html')
        if not t:
            break
        topics.extend(t)
        sleep(config.delay_between_requests)
    return set(topics)


def _grab_messages(topic):
    print str(topic),
    text = requests.get(_url_print_page.format(topic)).text
    selector = Selector(text=text)
    recomp = re.compile(r'<b>(?P<author>.*?)</b>\s\s(?P<day>[\d]{1,2}).(?P<month>[\d]{1,2}).(?P<year>[\d]{4}),'
                        r'\s(?P<hour>[\d]{1,2}):(?P<minute>[\d]{1,2})</h4>',
                        re.DOTALL)
    msgs = []
    divs = selector.xpath(_xpath_messages).extract()
    for div in divs:
        data = recomp.search(div)
        if data:
            # not every message has an id
            attachment = re.search('<!--IBF.ATTACHMENT_(?P<id>[\d]+)', div)
            msgs.append({
                'id': int(attachment.group('id')) if attachment else None,
                'author': data.group('author'),
                'created_at': str(datetime(int(data.group('year')), int(data.group('month')), int(data.group('day')),
                                           int(data.group('hour')), int(data.group('minute')))),
            })
    print str(len(msgs)) + ' messages'
    return msgs


def main():
    for fid in db.get_forum_ids():
        topics = _grab_topics(fid)
        for topic in topics:
            msgs = _grab_messages(topic)
            db.process_topic(topic, msgs)
            sleep(config.delay_between_requests)
<commit_msg>changed the way of collecting topics<commit_after># -*- coding: utf-8 -*-
import re
import db
import config
import requests
from time import sleep
from parsel import Selector
from datetime import datetime

_xpath_topics = '//*[@id="ipbcontent"]/div/ol/li'
_xpath_messages = '//*[@id="print"]/div[@class="printpost"]'
_url_lofi_page = 'http://diesel.elcat.kg/lofiversion/index.php?f{0}-{1}.html'
_url_print_page = 'http://diesel.elcat.kg/index.php?act=Print&client=printer&f=1&t={0}'


def _grab_topics(fid):
    topics = []
    recomp = re.compile(r'\?t(?P<id>[\d]+).html\'>(?P<title>.*?)</a>', re.DOTALL)
    ids = []
    for page in range(config.forum_pages_depth):
        print str(fid) + ':' + str(page + 1)
        # lite version topics list contains 150 topics per page
        text = requests.get(_url_lofi_page.format(fid, 150 * page)).text

        for topic in recomp.finditer(text):
            id = int(topic.group('id'))
            if id not in ids:
                ids.append(id)
                topics.append({
                    'id': id,
                    'title': topic.group('title'),
                })
    return topics


def _grab_messages(topic):
    print str(topic),
    text = requests.get(_url_print_page.format(topic)).text
    selector = Selector(text=text)
    recomp = re.compile(r'<b>(?P<author>.*?)</b>\s\s(?P<day>[\d]{1,2}).(?P<month>[\d]{1,2}).(?P<year>[\d]{4}),'
                        r'\s(?P<hour>[\d]{1,2}):(?P<minute>[\d]{1,2})</h4>',
                        re.DOTALL)
    msgs = []
    divs = selector.xpath(_xpath_messages).extract()
    for div in divs:
        data = recomp.search(div)
        if data:
            # not every message has an id
            attachment = re.search('<!--IBF.ATTACHMENT_(?P<id>[\d]+)', div)
            msgs.append({
                'id': int(attachment.group('id')) if attachment else None,
                'author': data.group('author'),
                'created_at': str(datetime(int(data.group('year')), int(data.group('month')), int(data.group('day')),
                                           int(data.group('hour')), int(data.group('minute')))),
            })
    print str(len(msgs)) + ' messages'
    return msgs


def main():
    for fid in db.get_forum_ids():
        topics = _grab_topics(fid)
        for topic in topics:
            msgs = _grab_messages(topic['id'])
            db.process_topic(topic, fid, msgs)
            sleep(config.delay_between_requests)

{'fraction_non_alphanumeric': 0.17398648648648649, 'fraction_numerical': 0.009290540540540541, 'mean_word_length': 2.4551422319474834, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2294129', 'n_tokens_mistral': 1640, 'n_tokens_neox': 1603, 'n_words': 289}