<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
from BeautifulSoup import BeautifulSoup
import re
import urllib2
import codecs

def fetch_data():
    return urllib2.urlopen("http://www.yle.fi/tekstitv/txt/P100_01.html").read()

def print_http_headers(content_length):
    print "\n".join((
                     "Expires: -1",
                     "Cache-Control: private, max-age=0",
                     "Content-Type: text/html; charset=UTF-8",
                     "Content-Length: %d" % content_length,
                   ))
    print "\n"

response_text = "";
soup = BeautifulSoup(fetch_data())
bigs = soup.findAll('big')
for big in bigs:
    m = re.search('<big><a href="([^"]+)">(\d+)</a>(.+)</big>', str(big))
    if m:
        href = m.group(1)
        page = m.group(2)
        title = m.group(3)
        response_text += '<div class="item"><a href="%s">%s</a> %s</div>\n' % (href, page, title)


print_http_headers(len(response_text))
print response_text
<commit_msg>Removed unnecessary import.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
from BeautifulSoup import BeautifulSoup
import re
import urllib2

def fetch_data():
    return urllib2.urlopen("http://www.yle.fi/tekstitv/txt/P100_01.html").read()

def print_http_headers(content_length):
    print "\n".join((
                     "Expires: -1",
                     "Cache-Control: private, max-age=0",
                     "Content-Type: text/html; charset=UTF-8",
                     "Content-Length: %d" % content_length,
                   ))
    print "\n"

response_text = "";
soup = BeautifulSoup(fetch_data())
bigs = soup.findAll('big')
for big in bigs:
    m = re.search('<big><a href="([^"]+)">(\d+)</a>(.+)</big>', str(big))
    if m:
        href = m.group(1)
        page = m.group(2)
        title = m.group(3)
        response_text += '<div class="item"><a href="%s">%s</a> %s</div>\n' % (href, page, title)


print_http_headers(len(response_text))
print response_text

{'fraction_non_alphanumeric': 0.1818642350557244, 'fraction_numerical': 0.014184397163120567, 'mean_word_length': 2.50177304964539, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '426987', 'n_tokens_mistral': 698, 'n_tokens_neox': 673, 'n_words': 132}