<commit_before>import functools
import time
import logging

import schedule

from .fetcher import fetch, cleanup_fetchers
from .storage import report_changes
from .notifier import notify
from .conf import settings


logger = logging.getLogger(__name__)


def main(once=False, log_level=logging.INFO):
    logging.getLogger("").setLevel(log_level)
    logger.debug("Arguments: %r",
                 {"once": once, "log_level": log_level})
    logger.debug("Configration: %r", settings.pages)
    try:
        check_all_pages(settings.pages)
        if not once:
            schedule_checks(settings.pages)
            logger.info("Starting infinite loop")
            while True:
                schedule.run_pending()
                time.sleep(60)
    finally:
        cleanup_fetchers()


def schedule_checks(page_confs):
    for conf in page_confs:
        period = conf.get("period", 300)
        logger.info(
            "Scheduling checks for %r every %r seconds",
            conf["name"],
            period,
        )
        schedule.every(period).seconds.do(
            functools.partial(check_page, conf)
        )


def check_all_pages(page_confs):
    for conf in page_confs:<commit_msg>Reducing log records length<commit_after>import time
import logging

import schedule

from .fetcher import fetch, cleanup_fetchers
from .storage import report_changes
from .notifier import notify
from .conf import settings


logger = logging.getLogger(__name__)


def main(once=False, log_level=logging.INFO):
    logging.getLogger("").setLevel(log_level)
    logger.debug("Arguments: %r",
                 {"once": once, "log_level": log_level})
    logger.debug("Configration: %r", settings.pages)
    try:
        check_all_pages(settings.pages)
        if not once:
            schedule_checks(settings.pages)
            logger.info("Starting infinite loop")
            while True:
                schedule.run_pending()
                time.sleep(60)
    finally:
        cleanup_fetchers()


def schedule_checks(page_confs):
    for conf in page_confs:
        period = conf.get("period", 300)
        logger.info(
            "Scheduling checks for %r every %r seconds",
            conf["name"],
            period,
        )
        schedule.every(period).seconds.do(
            lambda conf=conf: check_page(conf)
        )


def check_all_pages(page_confs):
    for conf in page_confs:
{'fraction_non_alphanumeric': 0.09666666666666666, 'fraction_numerical': 0.004166666666666667, 'mean_word_length': 2.489825581395349, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1956701', 'n_tokens_mistral': 702, 'n_tokens_neox': 672, 'n_words': 167}