<commit_before>import sys

import searx.search
import searx.search.processors
import searx.search.checker


if sys.stdout.isatty():
    RESET_SEQ = "\033[0m"
    COLOR_SEQ = "\033[1;%dm"
    BOLD_SEQ = "\033[1m"
    BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = map(lambda i: COLOR_SEQ % (30 + i), range(8))
else:
    RESET_SEQ = ""
    COLOR_SEQ = ""
    BOLD_SEQ = ""
    BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = "", "", "", "", "", "", "", ""


def iter_processor():
    if len(sys.argv) > 1:
        for name, processor in searx.search.processors.items():
            if name in sys.argv:
                yield name, processor
    else:
        for name, processor in searx.search.processors.items():
            yield name, processor


def main():
    searx.search.initialize()
    broken_urls = []
    for name, processor in iter_processor():
        if sys.stdout.isatty():
            print(BOLD_SEQ, 'Engine ', '%-30s' % name, RESET_SEQ, WHITE, ' Checking', RESET_SEQ)
        checker = searx.search.checker.Checker(processor)
        checker.run()
        if checker.test_results.succesfull:<commit_msg>[fix] do not colorize output on dumb terminals<commit_after>import sys
import os

import searx.search
import searx.search.processors
import searx.search.checker


if sys.stdout.isatty() and os.environ.get('TERM') not in ['dumb', 'unknown']:
    RESET_SEQ = "\033[0m"
    COLOR_SEQ = "\033[1;%dm"
    BOLD_SEQ = "\033[1m"
    BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = map(lambda i: COLOR_SEQ % (30 + i), range(8))
else:
    RESET_SEQ = ""
    COLOR_SEQ = ""
    BOLD_SEQ = ""
    BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = "", "", "", "", "", "", "", ""


def iter_processor():
    if len(sys.argv) > 1:
        for name, processor in searx.search.processors.items():
            if name in sys.argv:
                yield name, processor
    else:
        for name, processor in searx.search.processors.items():
            yield name, processor


def main():
    searx.search.initialize()
    broken_urls = []
    for name, processor in iter_processor():
        if sys.stdout.isatty():
            print(BOLD_SEQ, 'Engine ', '%-30s' % name, RESET_SEQ, WHITE, ' Checking', RESET_SEQ)
        checker = searx.search.checker.Checker(processor)
        checker.run()
        if checker.test_results.succesfull:
{'fraction_non_alphanumeric': 0.14454277286135694, 'fraction_numerical': 0.015170670037926675, 'mean_word_length': 2.7563291139240507, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1534848', 'n_tokens_mistral': 877, 'n_tokens_neox': 809, 'n_words': 198}