<commit_before>try:
    from http.server import SimpleHTTPRequestHandler, HTTPServer
except ImportError:  # Python 2
    from SimpleHTTPServer import SimpleHTTPRequestHandler
    from BaseHTTPServer import HTTPServer

import os
from multiprocessing import Process

import http_crawler


serving = False


def serve():
    global serving

    if serving:
        return

    serving = True

    def _serve(dir, port):
        base_dir = os.path.join('tests', dir)
        os.chdir(base_dir)
        server = HTTPServer(('', port), SimpleHTTPRequestHandler)
        server.serve_forever()

    Process(target=_serve, args=('site', 8000), daemon=True).start()
    Process(target=_serve, args=('external-site', 8001), daemon=True).start()


def test_crawl():
    serve()

    rsps = list(http_crawler.crawl('http://localhost:8000/'))

    assert len(rsps) == 13

    urls = [rsp.url for rsp in rsps]
<commit_msg>The daemon argument was only added in Python 3.3<commit_after>try:
    from http.server import SimpleHTTPRequestHandler, HTTPServer
except ImportError:  # Python 2
    from SimpleHTTPServer import SimpleHTTPRequestHandler
    from BaseHTTPServer import HTTPServer

import os
from multiprocessing import Process

import http_crawler


serving = False


def serve():
    global serving

    if serving:
        return

    serving = True

    def _serve(dir, port):
        base_dir = os.path.join('tests', dir)
        os.chdir(base_dir)
        server = HTTPServer(('', port), SimpleHTTPRequestHandler)
        server.serve_forever()

    proc_site = Process(target=_serve, args=('site', 8000))
    proc_site.daemon = True
    proc_site.start()

    proc_external_site = Process(target=_serve, args=('external-site', 8001))
    proc_external_site.daemon = True
    proc_external_site.start()


def test_crawl():
    serve()

    rsps = list(http_crawler.crawl('http://localhost:8000/'))

    assert len(rsps) == 13

    urls = [rsp.url for rsp in rsps]

{'fraction_non_alphanumeric': 0.10147883732789394, 'fraction_numerical': 0.01631820499745028, 'mean_word_length': 3.469248291571754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5079651', 'n_tokens_mistral': 644, 'n_tokens_neox': 603, 'n_words': 156}