<commit_before># -*- coding: utf-8 -*-
import multiprocessing
import time

import requests

from pinger.types import Response, InvalidContent, InvalidStatusCode, Timeout
from pinger.ext import ActionProvider


def watcher(url, expected_content, expected_status_code, interval, timeout, queue):
    response = Response(multiprocessing.current_process().name)

    try:
        request_response = requests.get(url, timeout=timeout)
    except requests.exceptions.ReadTimeout:
        error = Timeout('Open page', 'Timed out after {} seconds'.format(timeout))
        response.add_error(error)
    else:
        response.set_elapsed_time(request_response.elapsed)

        if expected_status_code != request_response.status_code:
            error = InvalidStatusCode(expected_result=expected_status_code, actual_result=request_response.status_code)
            response.add_error(error)

        if expected_content not in request_response.text:
            error = InvalidContent(expected_result=expected_content, actual_result='Content of {}'.format(url))
            response.add_error(error)

    queue.put(response.to_dict())
    time.sleep(interval)
    return watcher(url, expected_content, expected_status_code, interval, timeout, queue)


def post_processor(queue):
    """
    Processes results from watcher. Expects the result to be a dict
    containing the following data:

    ==========  ==========================================================
    status      Wether the request was successful or not
    errors      List of errors, each one being a dictionary itself.
    elapsed     Time taken for the request to be done
    ==========  ==========================================================
    """
    while True:
        response = queue.get()
        for Plugin in ActionProvider.plugins:
            plugin = Plugin()
            plugin.receive(status=response['status'], errors=response['errors'], elapsed=response['elapsed'])
<commit_msg>Add watcher documentation<commit_after># -*- coding: utf-8 -*-
import multiprocessing
import time

import requests

from pinger.types import Response, InvalidContent, InvalidStatusCode, Timeout
from pinger.ext import ActionProvider


def watcher(url, expected_content, expected_status_code, interval, timeout, queue):
    """
    Makes a request and validates the response. Response is sent to given queue to be
    processed afterwards by a different worker
    """
    response = Response(multiprocessing.current_process().name)

    try:
        request_response = requests.get(url, timeout=timeout)
    except requests.exceptions.ReadTimeout:
        error = Timeout('Open page', 'Timed out after {} seconds'.format(timeout))
        response.add_error(error)
    else:
        response.set_elapsed_time(request_response.elapsed)

        if expected_status_code != request_response.status_code:
            error = InvalidStatusCode(expected_result=expected_status_code, actual_result=request_response.status_code)
            response.add_error(error)

        if expected_content not in request_response.text:
            error = InvalidContent(expected_result=expected_content, actual_result='Content of {}'.format(url))
            response.add_error(error)

    queue.put(response.to_dict())
    time.sleep(interval)
    return watcher(url, expected_content, expected_status_code, interval, timeout, queue)


def post_processor(queue):
    """
    Processes results from watcher. Expects the result to be a dict
    containing the following data:

    ==========  ==========================================================
    status      Wether the request was successful or not
    errors      List of errors, each one being a dictionary itself.
    elapsed     Time taken for the request to be done
    ==========  ==========================================================
    """
    while True:
        response = queue.get()
        for Plugin in ActionProvider.plugins:
            plugin = Plugin()
            plugin.receive(status=response['status'], errors=response['errors'], elapsed=response['elapsed'])

{'fraction_non_alphanumeric': 0.14334554334554334, 'fraction_numerical': 0.0004884004884004884, 'mean_word_length': 3.779463243873979, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6371362', 'n_tokens_mistral': 1042, 'n_tokens_neox': 998, 'n_words': 281}