<commit_before>import sys
import sqlite3
import urllib.parse

from datetime import datetime

from lxml.html import parse


INDEX_INDEX = 'http://tvtropes.org/pmwiki/index_report.php'
WIKI_PAGE = 'http://tvtropes.org/pmwiki/pmwiki.php/'


def list_pages(namespace_url=None):
    list_url = namespace_url or INDEX_INDEX
    print('Crawling {}'.format(list_url))
    tree = parse(list_url)

    for a in tree.xpath('//a[@class="twikilink"]'):
        name = a.text.strip()
        url = a.attrib['href']
        if namespace_url:
            yield (name,), url
        else:
            yield ('Main', name), url

    if not namespace_url:
        namespaces = tree.xpath(
            '//a[starts-with(@href, "index_report.php?groupname=")]'
        )

        for a in namespaces:
            namespace = a.text.strip()
            url = urllib.parse.urljoin(
                INDEX_INDEX, a.attrib['href']
            )
            for key, value in list_pages(url):
                assert len(key) == 1
                yield (namespace,) + key, value


def save_links(links, cur):
    cur.execute('''
        CREATE TABLE IF NOT EXISTS indexindex
        (
        namespace text, name text, url text, last_crawled timestamp,
        constraint pk_indexindex primary key(namespace, name)
        )
    ''')
    for name, url in links:
        try:
            cur.execute('INSERT INTO indexindex VALUES (?, ?, ?, NULL)',
                        (name[0], name[1], url))
        except sqlite3.IntegrityError:
            cur.execute('UPDATE indexindex SET url = ? '
                        'WHERE namespace = ? and name = ?',
                        (url, name[0], name[1]))
    print('Total {}'.format(cur.execute('SELECT count(*) FROM indexindex')
                            .fetchone()[0]))


def links_to_crawl(crawl_stack):
    while crawl_stack:
        link = crawl_stack.pop()
        if len(link) == 3:
            yield link + (None,)
        else:
            yield link


def crawl_links(crawl_stack, conn):
    c = conn.cursor()
    c.execute('''
        CREATE TABLE IF NOT EXISTS relations
        (
        origin_namespace text, origin text,
        destination_namespace text, destination text,
        constraint pk_relations primary key(origin_namespace, origin,
        destination_namespace, destination)
        )
        ''')
    start_time = datetime.now()
    start_indexindex_count = int(
        c.execute('SELECT count(*) FROM indexindex')
        .fetchone()[0])
    start_relations_count = int(
        c.execute('SELECT count(*) FROM relations')
        .fetchone()[0])
    round_count = 0
    for namespace, name, url, referer in links_to_crawl(crawl_stack):
        current_time = datetime.now()
        tree = parse(url)
        if namespace is None:
            try:
                namespace = tree.xpath('//div[@class="pagetitle"]')[0] \
                    .text.strip()[:-1]
            except (AttributeError, AssertionError, IndexError):
                print('Error: There is no pagetitle on this page.')
                continue
            if namespace == '':
                namespace = 'Main'
        print("Fetching: {}/{} @ {}"
              .format(namespace, name, url))
        if c.execute('SELECT count(*) FROM indexindex '
                     'WHERE namespace = ? and name = ?',
                     (namespace, name)).fetchone()[0] != 0:
            last_crawled = c.execute('SELECT last_crawled FROM indexindex '
                                     'WHERE namespace = ? and name = ?',
                                     (namespace, name)).fetchone()
            if last_crawled is not None:
                if last_crawled[0] is not None:
                    if (current_time - last_crawled[0]).days < 7:
                        print('Skipping: {}/{} @ {} due to recent crawl '
                              'in 7 days'
                              .format(namespace, name, url))
                        continue
        else:
            try:
                c.execute('INSERT INTO indexindex VALUES (?, ?, ?, ?)',
                          (namespace, name, url, None))
            except sqlite3.IntegrityError:
                pass
        conn.commit()
        for a in tree.xpath('//a[@class="twikilink"]'):
            try:
                destination_name = a.text.strip()
                destination_url = urllib.parse.urljoin(
                    WIKI_PAGE, a.attrib['href']
                )
                next_crawl = (None, destination_name, destination_url,
                              (namespace, name))
                if next_crawl not in crawl_stack:<commit_msg>Relations now retrieve original title<commit_after>import sys
import sqlite3
import urllib.parse

from datetime import datetime

from lxml.html import parse


INDEX_INDEX = 'http://tvtropes.org/pmwiki/index_report.php'
WIKI_PAGE = 'http://tvtropes.org/pmwiki/pmwiki.php/'


def list_pages(namespace_url=None):
    list_url = namespace_url or INDEX_INDEX
    print('Crawling {}'.format(list_url))
    tree = parse(list_url)

    for a in tree.xpath('//a[@class="twikilink"]'):
        name = a.text.strip()
        url = a.attrib['href']
        if namespace_url:
            yield (name,), url
        else:
            yield ('Main', name), url

    if not namespace_url:
        namespaces = tree.xpath(
            '//a[starts-with(@href, "index_report.php?groupname=")]'
        )

        for a in namespaces:
            namespace = a.text.strip()
            url = urllib.parse.urljoin(
                INDEX_INDEX, a.attrib['href']
            )
            for key, value in list_pages(url):
                assert len(key) == 1
                yield (namespace,) + key, value


def save_links(links, cur):
    cur.execute('''
        CREATE TABLE IF NOT EXISTS indexindex
        (
        namespace text, name text, url text, last_crawled timestamp,
        constraint pk_indexindex primary key(namespace, name)
        )
    ''')
    for name, url in links:
        try:
            cur.execute('INSERT INTO indexindex VALUES (?, ?, ?, NULL)',
                        (name[0], name[1], url))
        except sqlite3.IntegrityError:
            cur.execute('UPDATE indexindex SET url = ? '
                        'WHERE namespace = ? and name = ?',
                        (url, name[0], name[1]))
    print('Total {}'.format(cur.execute('SELECT count(*) FROM indexindex')
                            .fetchone()[0]))


def links_to_crawl(crawl_stack):
    while crawl_stack:
        link = crawl_stack.pop()
        if len(link) == 3:
            yield link + (None,)
        else:
            yield link


def crawl_links(crawl_stack, conn):
    c = conn.cursor()
    c.execute('''
        CREATE TABLE IF NOT EXISTS relations
        (
        origin_namespace text, origin text,
        destination_namespace text, destination text,
        constraint pk_relations primary key(origin_namespace, origin,
        destination_namespace, destination)
        )
        ''')
    start_time = datetime.now()
    start_indexindex_count = int(
        c.execute('SELECT count(*) FROM indexindex')
        .fetchone()[0])
    start_relations_count = int(
        c.execute('SELECT count(*) FROM relations')
        .fetchone()[0])
    round_count = 0
    for namespace, name, url, referer in links_to_crawl(crawl_stack):
        current_time = datetime.now()
        tree = parse(url)
        try:
            namespace = tree.xpath('//div[@class="pagetitle"]')[0] \
                .text.strip()[:-1]
        except (AttributeError, AssertionError, IndexError):
            print('Error: There is no pagetitle on this page.')
            continue
        if namespace == '':
            namespace = 'Main'
        name = tree.xpath('//div[@class="pagetitle"]/span')[0].text.strip()
        print("Fetching: {}/{} @ {}"
              .format(namespace, name, url))
        if c.execute('SELECT count(*) FROM indexindex '
                     'WHERE namespace = ? and name = ?',
                     (namespace, name)).fetchone()[0] != 0:
            last_crawled = c.execute('SELECT last_crawled FROM indexindex '
                                     'WHERE namespace = ? and name = ?',
                                     (namespace, name)).fetchone()
            if last_crawled is not None:
                if last_crawled[0] is not None:
                    if (current_time - last_crawled[0]).days < 7:
                        print('Skipping: {}/{} @ {} due to recent crawl '
                              'in 7 days'
                              .format(namespace, name, url))
                        continue
        else:
            try:
                c.execute('INSERT INTO indexindex VALUES (?, ?, ?, ?)',
                          (namespace, name, url, None))
            except sqlite3.IntegrityError:
                pass
        conn.commit()
        for a in tree.xpath('//a[@class="twikilink"]'):
            try:
                destination_name = a.text.strip()
                destination_url = urllib.parse.urljoin(
                    WIKI_PAGE, a.attrib['href']
                )
                next_crawl = (None, destination_name, destination_url,
                              (namespace, name))
                if next_crawl not in crawl_stack:
{'fraction_non_alphanumeric': 0.11551650235747964, 'fraction_numerical': 0.0046078011144449205, 'mean_word_length': 1.658974358974359, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2081586', 'n_tokens_mistral': 2693, 'n_tokens_neox': 2549, 'n_words': 633}