<commit_before># coding=utf-8

""" A collection of internal utility functions used by the package
"""

import json
import random

import requests
import requests_cache

from mapi.constants import *

# Setup requests caching
requests_cache.install_cache('.mapi', expire_after=604800)  # week


def request_json(url, parameters=None, body=None, headers=None, agent=None):
    """ Queries a url for json data

    Essentially just wraps requests to abstract return values and exceptions

    Note: Requests are cached using requests_cached for a week, this is done
    transparently by using the package's monkey patching

    :param str url: The url to query
    :param dict parameters: Query string parameters
    :param dict body: JSON body parameters; if set implicitly POSTs
    :param headers: HTTP headers; content type, length, and user agent
        already get set internally
    :param str agent: User agent handle to include in headers; must be one
        specified in the constants module; if unset on will be chosen randomly
    :return: a list where the first item is the numeric status code and the
        second is a dict containing the JSON data retrieved
    """
    assert url
    status = 400

    if isinstance(headers, dict):
        headers = clean_dict(headers)
    if isinstance(parameters, dict):
        parameters = clean_dict(parameters)
    if body:
        method = 'POST'
        if isinstance(body, str):
            body = body.encode()
        elif isinstance(body, dict):
            body = json.dumps(body).encode()
        headers = headers or {}
        headers['content-type'] = 'application/json'
        headers['content-length'] = len(body)
        headers['user-agent'] = get_user_agent(agent)
    else:
        method = 'GET'
    try:
        response = requests.request(
            url=url,
            params=parameters,
            json=body,
            headers=headers,
            method=method,
        )
        status = response.status_code
        content = response.json() if status // 100 == 2 else None
    except (requests.RequestException, ValueError):
        content = None
    return status, content


def clean_dict(d):
    """ Convenience function that removes a dicts keys that have falsy values

    :param dict d: the dict to clean
    :return: the cleaned dict
    """
    assert isinstance(d, dict)
    return {
        str(k).strip(): str(v).strip()
        for k, v in d.items()
        if v not in (None, Ellipsis, [], (), '')
    }


def filter_meta(entries, max_hits=None, year=None, year_delta=None):
    """ Filters a list of metadata dicts

    Note: this might be a good candidate for future refactoring, perhaps to
    split them up?

    :param list entries: the list of metadata dicts
    :param int max_hits: the maximum number of entries to include in the list
    :param int or str year: the target year to filter around
    :param int year_delta: results will be filtered around this value inclusively
    :return: the filtered list of entries
    """
    assert isinstance(entries, list)

    def year_diff(x):
        return abs(int(x['year']) - int(year))

    # Remove duplicate entries
    unique_entries = list()
    [unique_entries.append(e) for e in entries if e not in unique_entries]
    entries = unique_entries<commit_msg>Adds logging to request_json<commit_after># coding=utf-8

""" A collection of internal utility functions used by the package
"""

import json
import random

import requests
import requests_cache

from mapi import log
from mapi.constants import *

# Setup requests caching
requests_cache.install_cache('.mapi', expire_after=604800)  # week


def request_json(url, parameters=None, body=None, headers=None, agent=None):
    """ Queries a url for json data

    Essentially just wraps requests to abstract return values and exceptions

    Note: Requests are cached using requests_cached for a week, this is done
    transparently by using the package's monkey patching

    :param str url: The url to query
    :param dict parameters: Query string parameters
    :param dict body: JSON body parameters; if set implicitly POSTs
    :param optional dict headers: HTTP headers; content type, length, and user
        agent already get set internally
    :param str agent: User agent handle to include in headers; must be one
        specified in the constants module; if unset on will be chosen randomly
    :return: a list where the first item is the numeric status code and the
        second is a dict containing the JSON data retrieved
    :rtype: list
    """
    assert url
    status = 400
    log.info("url: %s" % url)

    if isinstance(headers, dict):
        headers = clean_dict(headers)
        log.info("headers: %s" % headers)
    if isinstance(parameters, dict):
        parameters = clean_dict(parameters)
        log.info("parameters: %s" % parameters)
    if body:
        method = 'POST'
        if isinstance(body, str):
            body = body.encode()
        elif isinstance(body, dict):
            body = json.dumps(body).encode()
        headers = headers or {}
        headers['content-type'] = 'application/json'
        headers['content-length'] = len(body)
        headers['user-agent'] = get_user_agent(agent)
    else:
        method = 'GET'

    try:
        response = requests.request(
            url=url,
            params=parameters,
            json=body,
            headers=headers,
            method=method,
        )
        status = response.status_code
        content = response.json() if status // 100 == 2 else None
        log.debug("response: %s" % response)
    except (requests.RequestException, ValueError) as e:
        log.debug(e, exc_info=True)
        content = None

    log.info("method: %s" % method)
    log.info("status: %d" % status)
    log.info("content: %s" % content)
    return status, content


def clean_dict(d):
    """ Convenience function that removes a dicts keys that have falsy values

    :param dict d: the dict to clean
    :return: the cleaned dict
    :rtype: dict
    """
    assert isinstance(d, dict)
    return {
        str(k).strip(): str(v).strip()
        for k, v in d.items()
        if v not in (None, Ellipsis, [], (), '')
    }


def filter_meta(entries, max_hits=None, year=None, year_delta=None):
    """ Filters a list of metadata dicts

    Note: this might be a good candidate for future refactoring, perhaps to
    split them up?

    :param list entries: the list of metadata dicts
    :param int max_hits: the maximum number of entries to include in the list
    :param int or str year: the target year to filter around
    :param int year_delta: results are filtered around this value inclusively
    :return: the filtered list of entries
    :rtype: dict
    """
    assert isinstance(entries, list)

    def year_diff(x):
        return abs(int(x['year']) - int(year))

    # Remove duplicate entries
    unique_entries = list()
    [unique_entries.append(e) for e in entries if e not in unique_entries]
    entries = unique_entries
{'fraction_non_alphanumeric': 0.0843134476406405, 'fraction_numerical': 0.0039676916536772, 'mean_word_length': 2.7966648735879507, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '719294', 'n_tokens_mistral': 1948, 'n_tokens_neox': 1883, 'n_words': 762}