<commit_before>#!/usr/bin/python

import re
from bs4 import BeautifulSoup


def match(url):
    return re.match(r'^https?://forums.spacebattles.com/posts/\d+/?.*', url)

def extract(url, fetch):
    page = fetch(url)
    soup = BeautifulSoup(page, 'html5lib')

    base = soup.head.base.get('href')

    match = re.match(r'.+/posts/(\d+)/?', url)
    if not match:
        print("Unparseable post URL", url)
        return
    postid = match.group(1)

    story = {}
    story['title'] = str(soup.find('h1').string)
    story['author'] = str(soup.find('p', id='pageDescription').find('a', class_='username').string)

    post = post = soup.find('li', id='post-'+postid)
    links = post.find('blockquote', class_='messageText').find_all('a', class_='internalLink')
    if not links:
        print("No links in index?")

    chapters = []
    for link in links:
        href = link.get('href')
        if '/members/' in href:
            # skip links to users
            continue
        if not href.startswith('http'):
            href = base + href
        print("Extracting chapter from", href)
        match = re.match(r'.+#post-(\d+)$', href)
        if not match:
            match = re.match(r'.+/posts/(\d+)/?$', href)
            if not match:
                print("Unparseable index link href", href)
                return
        chapter_postid = match.group(1)
        chapter_page = fetch(href)
        chapter_soup = BeautifulSoup(chapter_page, 'html5lib')

        post = chapter_soup.find('li', id='post-'+chapter_postid).find('blockquote', class_='messageText')
        post.name = 'div'

        chapters.append((str(link.string), post.prettify()))

    story['chapters'] = chapters

    return story<commit_msg>Error handling and defaulting to the top post in a thread<commit_after>#!/usr/bin/python

import re
from bs4 import BeautifulSoup


def match(url):
    return re.match(r'^https?://forums.spacebattles.com/posts/\d+/?.*', url)

def extract(url, fetch):
    page = fetch(url)
    soup = BeautifulSoup(page, 'html5lib')

    base = soup.head.base.get('href')

    match = re.match(r'.+/posts/(\d+)/?', url)
    if not match:
        print("Unparseable post URL", url)
        return
    postid = match.group(1)

    story = {}
    story['title'] = str(soup.find('h1').string)
    story['author'] = str(soup.find('p', id='pageDescription').find('a', class_='username').string)

    post = post = soup.find('li', id='post-'+postid)
    links = post.find('blockquote', class_='messageText').find_all('a', class_='internalLink')
    if not links:
        print("No links in index?")

    chapters = []
    for link in links:
        href = link.get('href')
        if '/members/' in href:
            # skip links to users
            continue
        if not href.startswith('http'):
            href = base + href
        print("Extracting chapter from", href)
        match = re.match(r'.+#post-(\d+)$', href)
        if not match:
            match = re.match(r'.+/posts/(\d+)/?$', href)
            if not match:
                print("Unparseable index link href", href)
        chapter_postid = match and match.group(1)
        chapter_page = fetch(href)
        chapter_soup = BeautifulSoup(chapter_page, 'html5lib')

        if chapter_postid:
            post = chapter_soup.find('li', id='post-'+chapter_postid)
        else:
            # just the first one in the thread, then
            post = chapter_soup.find('li', class_='message')
        post = post.find('blockquote', class_='messageText')
        post.name = 'div'

        chapters.append((str(link.string), post.prettify()))

    story['chapters'] = chapters

    return story
{'fraction_non_alphanumeric': 0.1592270005443658, 'fraction_numerical': 0.00326619488296135, 'mean_word_length': 2.4933460076045626, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4205051', 'n_tokens_mistral': 1186, 'n_tokens_neox': 1152, 'n_words': 266}