<commit_before># -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 <NAME> and <NAME>
# Copyright (C) 2012-2013 <NAME>
import tempfile
import shutil
import re
import os
import multiprocessing
import urlparse
from itertools import islice
from unittest import TestCase
from dosagelib import scraper


def get_host(url):
    """Get host part of URL."""
    return urlparse.urlsplit(url)[1].lower()


# Dictionary with per-host locks.
_locks = {}
# Allowed number of connections per host
MaxConnections = 4

def get_lock(host):
    """Get bounded semphore for given host."""
    if host not in _locks:
        _locks[host] = multiprocessing.BoundedSemaphore(MaxConnections)
    return _locks[host]


class _ComicTester(TestCase):
    """Basic comic test class."""
    scraperclass=None

    def setUp(self):
        if self.scraperclass is not None:
            self.name = self.scraperclass.getName()
            self.url = self.scraperclass.starter()
            # create a temporary directory for images
            self.tmpdir = tempfile.mkdtemp()

    def tearDown(self):
        shutil.rmtree(self.tmpdir)

    def get_saved_images(self):
        """Get saved images."""
        dirs = tuple(self.name.split('/'))
        return os.listdir(os.path.join(self.tmpdir, *dirs))

    def test_comic(self):
        if self.scraperclass is None:
            # only run subclasses<commit_msg>Initialize tmpdir.<commit_after># -*- coding: iso-8859-1 -*-
# Copyright (C) 2004-2005 <NAME> and <NAME>
# Copyright (C) 2012-2013 <NAME>
import tempfile
import shutil
import re
import os
import multiprocessing
import urlparse
from itertools import islice
from unittest import TestCase
from dosagelib import scraper


def get_host(url):
    """Get host part of URL."""
    return urlparse.urlsplit(url)[1].lower()


# Dictionary with per-host locks.
_locks = {}
# Allowed number of connections per host
MaxConnections = 4

def get_lock(host):
    """Get bounded semphore for given host."""
    if host not in _locks:
        _locks[host] = multiprocessing.BoundedSemaphore(MaxConnections)
    return _locks[host]


class _ComicTester(TestCase):
    """Basic comic test class."""
    scraperclass=None

    def setUp(self):
        if self.scraperclass is not None:
            self.name = self.scraperclass.getName()
            self.url = self.scraperclass.starter()
            # create a temporary directory for images
            self.tmpdir = tempfile.mkdtemp()
        else:
            self.tmpdir = None

    def tearDown(self):
        if self.tmpdir is not None:
            shutil.rmtree(self.tmpdir)

    def get_saved_images(self):
        """Get saved images."""
        dirs = tuple(self.name.split('/'))
        return os.listdir(os.path.join(self.tmpdir, *dirs))

    def test_comic(self):
        if self.scraperclass is None:
            # only run subclasses
{'fraction_non_alphanumeric': 0.1060711793440335, 'fraction_numerical': 0.016050244242847175, 'mean_word_length': 2.8848238482384825, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3059173', 'n_tokens_mistral': 934, 'n_tokens_neox': 878, 'n_words': 256}