<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
test_pycrawl
----------------------------------

Tests for `pycrawl` module.
"""

import errno
from functools import wraps
import os
import re
import shutil
import time
import unittest
try:  # Python 3
    from urllib.parse import urlparse
except:  # Python 2
    from urlparse import urlparse

import bs4

from tests.run_server import run_server, stop_server
from pycrawl import pycrawl


def run_main_with_url(url):
    """Run main on the given URL, then delete the downloaded files after."""
    def decorator(f):
        @wraps(f)
        def wrapper(*args, **kwargs):
            argv = ['pycrawl.py', url]
            try:
                pycrawl.main(argv)
                f(*args, **kwargs)
            finally:
                # remove created download dir
                try:
                    basename = urlparse(url).hostname
                    shutil.rmtree(basename)
                except OSError as e:
                    if e.errno == errno.ENOENT:
                        pass
                    else:
                        raise
        return wrapper
    return decorator


class TestPycrawl(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.server = run_server()
        time.sleep(0.5)  # give it time to start up

    @classmethod
    def tearDownClass(cls):
        stop_server(cls.server)

    def setUp(self):
        pass

    def tearDown(self):
        pass

    @run_main_with_url('http://nonexistentsite.nes/badpath')
    def test_nonexistent_url(self):
        self.assertFalse(
            os.path.isfile('nonexistentsite.nes/badpath'))

    @run_main_with_url('http://localhost:8000/Python_logo_100x100.jpg')
    def test_non_html_url(self):
        self.assertTrue(os.path.isfile('localhost/Python_logo_100x100.jpg'))

    @run_main_with_url('http://localhost:8000')
    def test_valid_url_downloads_site(self):
        with open('localhost/__root__') as f:
            root = bs4.BeautifulSoup(f)
        self.assertTrue(bool(root.find(text=re.compile("Text"))))
        # now try a page linked from the first
        with open('localhost/local-relative.html') as f:
            relative = bs4.BeautifulSoup(f)
        self.assertTrue(bool(relative.find(text=re.compile("relative"))))


if __name__ == '__main__':
    unittest.main()
<commit_msg>absolute local link test added, already passing<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
test_pycrawl
----------------------------------

Tests for `pycrawl` module.
"""

import errno
from functools import wraps
import os
import re
import shutil
import time
import unittest
try:  # Python 3
    from urllib.parse import urlparse
except:  # Python 2
    from urlparse import urlparse

import bs4

from tests.run_server import run_server, stop_server
from pycrawl import pycrawl


def run_main_with_url(url):
    """Run main on the given URL, then delete the downloaded files after."""
    def decorator(f):
        @wraps(f)
        def wrapper(*args, **kwargs):
            argv = ['pycrawl.py', url]
            try:
                pycrawl.main(argv)
                f(*args, **kwargs)
            finally:
                # remove created download dir
                try:
                    basename = urlparse(url).hostname
                    shutil.rmtree(basename)
                except OSError as e:
                    if e.errno == errno.ENOENT:
                        pass
                    else:
                        raise
        return wrapper
    return decorator


class TestPycrawl(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.server = run_server()
        time.sleep(0.5)  # give it time to start up

    @classmethod
    def tearDownClass(cls):
        stop_server(cls.server)

    def setUp(self):
        pass

    def tearDown(self):
        pass

    @run_main_with_url('http://nonexistentsite.nes/badpath')
    def test_nonexistent_url(self):
        self.assertFalse(
            os.path.isfile('nonexistentsite.nes/badpath'))

    @run_main_with_url('http://localhost:8000/Python_logo_100x100.jpg')
    def test_non_html_url(self):
        self.assertTrue(os.path.isfile('localhost/Python_logo_100x100.jpg'))

    @run_main_with_url('http://localhost:8000')
    def test_valid_url_downloads_site(self):
        with open('localhost/__root__') as f:
            root = bs4.BeautifulSoup(f)
        self.assertTrue(bool(root.find(text=re.compile("Text"))))
        # now try a page linked from the first
        with open('localhost/local-relative.html') as f:
            relative = bs4.BeautifulSoup(f)
        self.assertTrue(bool(relative.find(text=re.compile("relative"))))
        # and one linked via an explicit local domain
        with open('localhost/local-explicit.html') as f:
            explicit = bs4.BeautifulSoup(f)
        self.assertTrue(bool(explicit.find(text=re.compile("explicit"))))


if __name__ == '__main__':
    unittest.main()

{'fraction_non_alphanumeric': 0.11507067489548078, 'fraction_numerical': 0.011347800119450527, 'mean_word_length': 2.1537978656622725, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '621876', 'n_tokens_mistral': 1595, 'n_tokens_neox': 1492, 'n_words': 345}