<commit_before>#!/usr/bin/env python

import glob
import logging
import os
import re
import shutil
import unittest
import uuid
import urlparse

from httmock import urlmatch, HTTMock
import requests
import yaml

import harprofiler
import haruploader


# Override logging level for tests
loggers = [
    logging.getLogger('haruploader'),
    logging.getLogger('harprofiler'),
]

for log in loggers:
    log.setLevel(logging.WARNING)


class ProfilerTest(unittest.TestCase):

    def test_run_cached_property(self):
        url = 'https://www.edx.org/'
        config = yaml.load(file('test_config.yaml'))
        profiler = harprofiler.HarProfiler(config, url)
        self.assertTrue(profiler.run_cached)

    def test_label_prefix(self):
        url = 'https://www.edx.org/'
        expected_label = 'testprefix-https-www-edx-org'
        config = yaml.load(file('test_config.yaml'))
        profiler = harprofiler.HarProfiler(config, url)
        self.assertEqual(profiler.label, expected_label)

    def test_cached_label_prefix(self):
        url = 'https://www.edx.org/'
        expected_label = 'testprefix-https-www-edx-org-cached'
        config = yaml.load(file('test_config.yaml'))
        profiler = harprofiler.HarProfiler(config, url)
        self.assertEqual(profiler.cached_label, expected_label)

    def test_blank_label_prefix(self):
        url = 'https://www.edx.org/'
        expected_label = 'https-www-edx-org-cached'
        config = yaml.load(file('test_config.yaml'))
        config['label_prefix'] = None
        profiler = harprofiler.HarProfiler(config, url)
        self.assertEqual(profiler.cached_label, expected_label)

    def test_slugify_simple_url(self):
        url = 'https://www.edx.org/'
        expected_slug = 'https-www-edx-org'
        config = yaml.load(file('test_config.yaml'))
        profiler = harprofiler.HarProfiler(config, url)
        slug = profiler.slugify(url)
        self.assertEqual(slug, expected_slug)

    def test_slugify_complex_url(self):
        url = 'https://www.edx.org/course/mitx/foo-2881#.VE6swYWFuR9'
        expected_slug = 'https-www-edx-org-course-mitx-foo-2881-ve6swywfur9'
        config = yaml.load(file('test_config.yaml'))
        profiler = harprofiler.HarProfiler(config, url)
        slug = profiler.slugify(url)
        self.assertEqual(slug, expected_slug)

    def test_default_config(self):
        cfg = yaml.load(file('test_config.yaml'))
        self.assertEqual(
            cfg['browsermob_dir'],
            './browsermob-proxy-2.0-beta-9'
        )
        self.assertTrue(cfg['run_cached'])
        self.assertTrue(cfg['virtual_display'])
        self.assertEqual(cfg['virtual_display_size_x'], 1024)
        self.assertEqual(cfg['virtual_display_size_y'], 768)


class UploaderTest(unittest.TestCase):

    def test_url_join_no_ending_slash(self):
        base_url = 'http://foo.bar.com'
        path = '/results/upload'
        url = urlparse.urljoin(base_url, path)
        self.assertEqual(url, 'http://foo.bar.com/results/upload')

    def test_url__join_ending_in_slash(self):
        base_url = 'http://foo.bar.com/'
        path = '/results/upload'
        url = urlparse.urljoin(base_url, path)
        self.assertEqual(url, 'http://foo.bar.com/results/upload')


class HarFileTestCase(unittest.TestCase):
    def setUp(self):
        self.config = yaml.load(file('test_config.yaml'))
        self.test_dir = self.config['har_dir']
        os.makedirs(self.test_dir)
        self.addCleanup(self.remove_hars)

    def remove_hars(self):
        shutil.rmtree(self.test_dir)


class AcceptanceTest(HarFileTestCase):
    def test_main(self):
        harprofiler.main('test_config.yaml')
        num_urls = len(self.config['urls'])
        num_pageloads = num_urls * 2  # uncached and cached
        num_hars = len(glob.glob(os.path.join(self.test_dir, '*.har')))
        self.assertEqual(num_hars, num_pageloads)


class StorageTest(HarFileTestCase):
    """
    Tests to confirm that the response handling for sending to harstorage works
    as expected.
    """
    def setUp(self):<commit_msg>reverted tests<commit_after>#!/usr/bin/env python

import glob
import logging
import os
import re
import shutil
import unittest
import uuid
import urlparse

from httmock import urlmatch, HTTMock
import requests
import yaml

import harprofiler
import haruploader


# Override logging level for tests
loggers = [
    logging.getLogger('haruploader'),
    logging.getLogger('harprofiler'),
]

for log in loggers:
    log.setLevel(logging.WARNING)


class ProfilerTest(unittest.TestCase):

    def test_run_cached_property(self):
        url = 'https://www.edx.org/'
        config = yaml.load(file('test_config.yaml'))
        profiler = harprofiler.HarProfiler(config, url)
        self.assertTrue(profiler.run_cached)

    def test_label_prefix(self):
        url = 'https://www.edx.org/'
        expected_label = 'testprefix-https-www-edx-org'
        config = yaml.load(file('test_config.yaml'))
        profiler = harprofiler.HarProfiler(config, url)
        self.assertEqual(profiler.label, expected_label)

    def test_cached_label_prefix(self):
        url = 'https://www.edx.org/'
        expected_label = 'testprefix-https-www-edx-org-cached'
        config = yaml.load(file('test_config.yaml'))
        profiler = harprofiler.HarProfiler(config, url)
        self.assertEqual(profiler.cached_label, expected_label)

    def test_blank_label_prefix(self):
        url = 'https://www.edx.org/'
        expected_label = 'https-www-edx-org-cached'
        config = yaml.load(file('test_config.yaml'))
        config['label_prefix'] = None
        profiler = harprofiler.HarProfiler(config, url)
        self.assertEqual(profiler.cached_label, expected_label)

    def test_slugify_simple_url(self):
        url = 'https://www.edx.org/'
        expected_slug = 'https-www-edx-org'
        config = yaml.load(file('test_config.yaml'))
        profiler = harprofiler.HarProfiler(config, url)
        slug = profiler.slugify(url)
        self.assertEqual(slug, expected_slug)

    def test_slugify_complex_url(self):
        url = 'https://www.edx.org/course/mitx/foo-2881#.VE6swYWFuR9'
        expected_slug = 'https-www-edx-org-course-mitx-foo-2881-ve6swywfur9'
        config = yaml.load(file('test_config.yaml'))
        profiler = harprofiler.HarProfiler(config, url)
        slug = profiler.slugify(url)
        self.assertEqual(slug, expected_slug)

    def test_default_config(self):
        cfg = yaml.load(file('test_config.yaml'))
        self.assertEqual(
            cfg['browsermob_dir'],
            './browsermob-proxy-2.0-beta-9'
        )
        self.assertTrue(cfg['run_cached'])
        self.assertTrue(cfg['virtual_display'])
        self.assertEqual(cfg['virtual_display_size_x'], 1024)
        self.assertEqual(cfg['virtual_display_size_y'], 768)


class HarFileTestCase(unittest.TestCase):
    def setUp(self):
        self.config = yaml.load(file('test_config.yaml'))
        self.test_dir = self.config['har_dir']
        os.makedirs(self.test_dir)
        self.addCleanup(self.remove_hars)

    def remove_hars(self):
        shutil.rmtree(self.test_dir)


class AcceptanceTest(HarFileTestCase):
    def test_main(self):
        harprofiler.main('test_config.yaml')
        num_urls = len(self.config['urls'])
        num_pageloads = num_urls * 2  # uncached and cached
        num_hars = len(glob.glob(os.path.join(self.test_dir, '*.har')))
        self.assertEqual(num_hars, num_pageloads)


class StorageTest(HarFileTestCase):
    """
    Tests to confirm that the response handling for sending to harstorage works
    as expected.
    """
    def setUp(self):
{'fraction_non_alphanumeric': 0.12077483099323973, 'fraction_numerical': 0.005980239209568383, 'mean_word_length': 3.087672688629118, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 12, 'lorem ipsum': 0, 'www.': 22, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5789783', 'n_tokens_mistral': 2525, 'n_tokens_neox': 2545, 'n_words': 402}