<commit_before># -*- coding: utf-8 -*-
"""
    captiveportal
    *************

    This test is a collection of tests to detect the presence of a
    captive portal. Code is taken, in part, from the old ooni-probe,
    which was written by <NAME> and <NAME>.

    :copyright: (c) 2012 <NAME>
    :license: see LICENSE for more details
"""
import base64
import os
import random
import re
import string
import urllib2
from urlparse import urlparse

from ooni import nettest
from ooni.templates import httpt
from ooni.utils import log

try:
    from dns import resolver
except ImportError:
    print "The dnspython module was not found. https://crate.io/packages/dnspython/"
    resolver = None

__plugoo__ = "captiveportal"
__desc__ = "Captive portal detection test"

optParameters = [['asset', 'a', None, 'Asset file'],
                 ['experiment-url', 'e', 'http://google.com/', 'Experiment URL'],
                 ['user-agent', 'u', random.choice(httpt.useragents),
                  'User agent for HTTP requests']
                ]

class CaptivePortal(nettest.TestCase):
    """
    Compares content and status codes of HTTP responses, and attempts
    to determine if content has been altered.
    """

    name = "captivep"
    description = "Captive Portal Test"
    requirements = None

    def http_fetch(self, url, headers={}):
        """
        Parses an HTTP url, fetches it, and returns a urllib2 response
        object.
        """
        url = urlparse(url).geturl()<commit_msg>* Cleaned up a statement printed on ImportError when dnspython is not   installed. * TODO Fix the Resolver in twisted.python.names so that we can use it instead   of dnspython.<commit_after># -*- coding: utf-8 -*-
"""
    captiveportal
    *************

    This test is a collection of tests to detect the presence of a
    captive portal. Code is taken, in part, from the old ooni-probe,
    which was written by <NAME> and <NAME>.

    :copyright: (c) 2012 <NAME>
    :license: see LICENSE for more details
"""
import base64
import os
import random
import re
import string
import urllib2
from urlparse import urlparse

from ooni import nettest
from ooni.templates import httpt
from ooni.utils import log

try:
    from dns import resolver
except ImportError:
    print "The dnspython module was not found:"
    print "See https://crate.io/packages/dnspython/"
    resolver = None

__plugoo__ = "captiveportal"
__desc__ = "Captive portal detection test"

optParameters = [['asset', 'a', None, 'Asset file'],
                 ['experiment-url', 'e', 'http://google.com/', 'Experiment URL'],
                 ['user-agent', 'u', random.choice(httpt.useragents),
                  'User agent for HTTP requests']
                ]

class CaptivePortal(nettest.TestCase):
    """
    Compares content and status codes of HTTP responses, and attempts
    to determine if content has been altered.
    """

    name = "captivep"
    description = "Captive Portal Test"
    requirements = None

    def http_fetch(self, url, headers={}):
        """
        Parses an HTTP url, fetches it, and returns a urllib2 response
        object.
        """
        url = urlparse(url).geturl()
{'fraction_non_alphanumeric': 0.10493245366006912, 'fraction_numerical': 0.005655042412818096, 'mean_word_length': 2.817745803357314, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6621361', 'n_tokens_mistral': 934, 'n_tokens_neox': 913, 'n_words': 350}