<commit_before>import unittest
import urllib
import os
import cgi
from wsgi_intercept.test_wsgi_app import simple_app

def build_url_for_file(name):
    return "file://%s/industrialise/tests/html/%s" % (os.getcwd(), name)

class TestBrowser(unittest.TestCase):

    def test_provide_cookiejar(self):
        from industrialise import browser
        sigil = object()
        b = browser.Browser(sigil)
        self.failUnless(sigil is b._cookiejar)

    def test_instantiate(self):
        from industrialise import browser
        browser.Browser()

    def test_load_data(self):
        from industrialise import browser
        b = browser.Browser()
        url = build_url_for_file("valid_html5.html")
        data, final_url, info = b._load_data(url)
        from_disk = open("industrialise/tests/html/valid_html5.html").read()
        self.assertEqual(data, from_disk)
        self.assertEqual(final_url, url)
        self.assertEqual(info['Content-length'], str(len(from_disk)))
<commit_msg>add test cookie<commit_after># -*- test-case-name: "industrialise.tests.test_browser" -*-

import unittest
import urllib
import os
import cgi
from wsgi_intercept.test_wsgi_app import simple_app

def build_url_for_file(name):
    return "file://%s/industrialise/tests/html/%s" % (os.getcwd(), name)

class TestBrowser(unittest.TestCase):

    def test_provide_cookiejar(self):
        from industrialise import browser
        sigil = object()
        b = browser.Browser(sigil)
        self.failUnless(sigil is b._cookiejar)

    def test_instantiate(self):
        from industrialise import browser
        browser.Browser()

    def test_load_data(self):
        from industrialise import browser
        b = browser.Browser()
        url = build_url_for_file("valid_html5.html")
        data, final_url, info = b._load_data(url)
        from_disk = open("industrialise/tests/html/valid_html5.html").read()
        self.assertEqual(data, from_disk)
        self.assertEqual(final_url, url)
        self.assertEqual(info['Content-length'], str(len(from_disk)))

{'fraction_non_alphanumeric': 0.10485436893203884, 'fraction_numerical': 0.001941747572815534, 'mean_word_length': 3.375796178343949, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3812951', 'n_tokens_mistral': 649, 'n_tokens_neox': 623, 'n_words': 129}