<commit_before># SPDX-License-Identifier: MIT
# Copyright (C) 2004-2008 <NAME> and <NAME>
# Copyright (C) 2012-2014 <NAME>
# Copyright (C) 2015-2022 <NAME>
import re
import os

import pytest
from xdist.dsession import LoadScopeScheduling

from dosagelib.scraper import scrapers


def get_test_scrapers():
    """Return scrapers that should be tested."""
    if 'TESTALL' in os.environ:
        # test all comics (this will take some time)
        return scrapers.get()
    elif 'TESTCOMICS' in os.environ:
        scraper_pattern = os.environ['TESTCOMICS']
    else:
        # Get limited number of scraper tests as default
        testscrapernames = [
            # "classic" _BasicScraper
            'AbstruseGoose',
            # complex _ParserScraper
            'GoComics/CalvinAndHobbes',
            # _WordPressScraper
            'GrrlPower',
        ]
        scraper_pattern = '^(' + '|'.join(testscrapernames) + ')$'

    matcher = re.compile(scraper_pattern)
    return [
        scraperobj for scraperobj in scrapers.get()
        if matcher.match(scraperobj.name)
    ]


def pytest_generate_tests(metafunc):
    if 'scraperobj' in metafunc.fixturenames:
        scrapers = get_test_scrapers()
        scraperids = [x.name for x in scrapers]
        metafunc.parametrize('scraperobj', scrapers, ids=sorted(scraperids))


class LoadModScheduling(LoadScopeScheduling):<commit_msg>REALLY make sure generated test order is stable<commit_after># SPDX-License-Identifier: MIT
# Copyright (C) 2004-2008 <NAME> and <NAME>
# Copyright (C) 2012-2014 <NAME>
# Copyright (C) 2015-2022 <NAME>
import re
import os
from operator import attrgetter

import pytest
from xdist.dsession import LoadScopeScheduling

from dosagelib.scraper import scrapers


def get_test_scrapers():
    """Return scrapers that should be tested."""
    if 'TESTALL' in os.environ:
        # test all comics (this will take some time)
        return scrapers.get()
    elif 'TESTCOMICS' in os.environ:
        scraper_pattern = os.environ['TESTCOMICS']
    else:
        # Get limited number of scraper tests as default
        testscrapernames = [
            # "classic" _BasicScraper
            'AbstruseGoose',
            # complex _ParserScraper
            'GoComics/CalvinAndHobbes',
            # _WordPressScraper
            'GrrlPower',
        ]
        scraper_pattern = '^(' + '|'.join(testscrapernames) + ')$'

    matcher = re.compile(scraper_pattern)
    return [
        scraperobj for scraperobj in scrapers.get()
        if matcher.match(scraperobj.name)
    ]


def pytest_generate_tests(metafunc):
    if 'scraperobj' in metafunc.fixturenames:
        scrapers = sorted(get_test_scrapers(), key=attrgetter('name'))
        metafunc.parametrize('scraperobj', scrapers, ids=attrgetter('name'))


class LoadModScheduling(LoadScopeScheduling):
{'fraction_non_alphanumeric': 0.09788732394366197, 'fraction_numerical': 0.016901408450704224, 'mean_word_length': 2.9734265734265732, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '209069', 'n_tokens_mistral': 955, 'n_tokens_neox': 920, 'n_words': 221}