<commit_before>import logging
import random
from time import sleep
import unittest
import hashlib
import os
from unittest.case import SkipTest

from prismriver.main import search
from prismriver.struct import SearchConfig


class TestPlugins(unittest.TestCase):
    def check_plugin(self, plugin_id, artist, title, lyric_hashes):
        logging.basicConfig(format='%(asctime)s %(levelname)s [%(module)s] %(message)s', level=logging.DEBUG)

        self.is_skipped(plugin_id)

        retry_delay = int(os.getenv('PRISMRIVER_TEST_RETRY_DELAY', '120'))
        retry_count = int(os.getenv('PRISMRIVER_TEST_RETRY_COUNT', '2'))

        config = self.get_search_config(plugin_id)

        step = 0
        result = search(artist, title, config)
        while not result and step < retry_count:
            delay = retry_delay * (step + 1)
            logging.debug('Empty search result, wait for {} sec. before retry'.format(delay))
            sleep(delay)
            result = search(artist, title, config)
            step += 1

        self.assertEqual(1, len(result), 'Wrong songs count')
        self.assertEqual(len(lyric_hashes), len(result[0].lyrics), 'Wrong lyrics count')

        index = 0
        for lyric_hash in lyric_hashes:
            logging.debug('Lyric #{}:\n'.format(index) + result[0].lyrics[index])
            self.assertEqual(lyric_hash, self.get_md5(result[0].lyrics[index]),
                             'Different text in #{} lyrics'.format(index))
            index += 1

    def is_skipped(self, plugin_id):
        disabled_plugins = os.getenv('PRISMRIVER_TEST_DISABLED_PLUGINS', '').split(',')
        if plugin_id in disabled_plugins:
            raise SkipTest('plugin test disabled')

    def setUp(self):
        max_delay = int(os.getenv('PRISMRIVER_TEST_MAX_DELAY', '0'))
        delay = random.randint(0, max_delay)
        logging.debug('Wait {}sec. before executing test...'.format(delay))
        sleep(delay)

    def get_md5(self, value):
        return hashlib.md5(value.encode('utf-8')).hexdigest()<commit_msg>Re-download pages on failed tests<commit_after>import logging
import random
from time import sleep
import unittest
import hashlib
import os
from unittest.case import SkipTest

from prismriver.main import search
from prismriver.struct import SearchConfig


class TestPlugins(unittest.TestCase):
    def check_plugin(self, plugin_id, artist, title, lyric_hashes):
        logging.basicConfig(format='%(asctime)s %(levelname)s [%(module)s] %(message)s', level=logging.DEBUG)

        self.is_skipped(plugin_id)

        retry_delay = int(os.getenv('PRISMRIVER_TEST_RETRY_DELAY', '120'))
        retry_count = int(os.getenv('PRISMRIVER_TEST_RETRY_COUNT', '2'))

        config = self.get_search_config(plugin_id)

        step = 0
        result = search(artist, title, config)
        while not result and step < retry_count:
            config.cache_web_ttl_sec = 0  # force test to re-download pages instead of getting them from cache
            delay = retry_delay * (step + 1)
            logging.debug('Empty search result, wait for {} sec. before retry'.format(delay))
            sleep(delay)
            result = search(artist, title, config)
            step += 1

        self.assertEqual(1, len(result), 'Wrong songs count')
        self.assertEqual(len(lyric_hashes), len(result[0].lyrics), 'Wrong lyrics count')

        index = 0
        for lyric_hash in lyric_hashes:
            logging.debug('Lyric #{}:\n'.format(index) + result[0].lyrics[index])
            self.assertEqual(lyric_hash, self.get_md5(result[0].lyrics[index]),
                             'Different text in #{} lyrics'.format(index))
            index += 1

    def is_skipped(self, plugin_id):
        disabled_plugins = os.getenv('PRISMRIVER_TEST_DISABLED_PLUGINS', '').split(',')
        if plugin_id in disabled_plugins:
            raise SkipTest('plugin test disabled')

    def setUp(self):
        max_delay = int(os.getenv('PRISMRIVER_TEST_MAX_DELAY', '0'))
        delay = random.randint(0, max_delay)
        logging.debug('Wait {}sec. before executing test...'.format(delay))
        sleep(delay)

    def get_md5(self, value):
        return hashlib.md5(value.encode('utf-8')).hexdigest()
{'fraction_non_alphanumeric': 0.11671775524640415, 'fraction_numerical': 0.009195944352746994, 'mean_word_length': 3.0018867924528303, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5275033', 'n_tokens_mistral': 1333, 'n_tokens_neox': 1281, 'n_words': 294}