<commit_before># coding=utf-8

""" A collection of utility functions non-specific to mapi's domain logic
"""

import random
from os.path import join
from re import match
from sys import version_info

import requests_cache
from appdirs import user_cache_dir
from requests.adapters import HTTPAdapter

from mapi import log
from mapi.compatibility import ustr

__all__ = [
    "AGENT_CHROME",
    "AGENT_EDGE",
    "AGENT_IOS",
    "AGENT_ALL",
    "CACHE_PATH",
    "clear_cache",
    "clean_dict",
    "d2l",
    "get_session",
    "get_user_agent",
    "request_json",
]

AGENT_CHROME = (
    "Mozilla/5.0 (iPhone; CPU iPhone OS 10_0_1 like Mac OS X) AppleWebKit/601.1"
    " (KHTML, like Gecko) CriOS/53.0.2785.86 Mobile/14A403 Safari/601.1.46"
)
AGENT_EDGE = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like "
    "Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393"
)
AGENT_IOS = (
    "Mozilla/5.0 (iPhone; CPU iPhone OS 10_0_1 like Mac OS X) "
    "AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A403 "
    "Safari/602.1"
)
AGENT_ALL = (AGENT_CHROME, AGENT_EDGE, AGENT_IOS)
CACHE_PATH = join(user_cache_dir(), "mapi-py%d.sqlite" % version_info.major)


def clear_cache():
    """ Clears requests-cache cache
    """
    get_session().cache.clear()


def clean_dict(target_dict, whitelist=None):
    """ Convenience function that removes a dicts keys that have falsy values
    """
    assert isinstance(target_dict, dict)
    return {
        ustr(k).strip(): ustr(v).strip()
        for k, v in target_dict.items()
        if v not in (None, Ellipsis, [], (), "")
        and (not whitelist or k in whitelist)
    }
<commit_msg>Adds year_expand to __all__<commit_after># coding=utf-8

""" A collection of utility functions non-specific to mapi's domain logic
"""

import random
from os.path import join
from re import match
from sys import version_info

import requests_cache
from appdirs import user_cache_dir
from requests.adapters import HTTPAdapter

from mapi import log
from mapi.compatibility import ustr

__all__ = [
    "AGENT_ALL",
    "AGENT_CHROME",
    "AGENT_EDGE",
    "AGENT_IOS",
    "CACHE_PATH",
    "clean_dict",
    "clear_cache",
    "d2l",
    "get_session",
    "get_user_agent",
    "request_json",
    "year_expand",
]

AGENT_CHROME = (
    "Mozilla/5.0 (iPhone; CPU iPhone OS 10_0_1 like Mac OS X) AppleWebKit/601.1"
    " (KHTML, like Gecko) CriOS/53.0.2785.86 Mobile/14A403 Safari/601.1.46"
)
AGENT_EDGE = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like "
    "Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393"
)
AGENT_IOS = (
    "Mozilla/5.0 (iPhone; CPU iPhone OS 10_0_1 like Mac OS X) "
    "AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A403 "
    "Safari/602.1"
)
AGENT_ALL = (AGENT_CHROME, AGENT_EDGE, AGENT_IOS)
CACHE_PATH = join(user_cache_dir(), "mapi-py%d.sqlite" % version_info.major)


def clean_dict(target_dict, whitelist=None):
    """ Convenience function that removes a dicts keys that have falsy values
    """
    assert isinstance(target_dict, dict)
    return {
        ustr(k).strip(): ustr(v).strip()
        for k, v in target_dict.items()
        if v not in (None, Ellipsis, [], (), "")
        and (not whitelist or k in whitelist)
    }


def clear_cache():
    """ Clears requests-cache cache
    """
    get_session().cache.clear()

{'fraction_non_alphanumeric': 0.1236417033773862, 'fraction_numerical': 0.05345080763582966, 'mean_word_length': 3.3948387096774195, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6458935', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1273, 'n_words': 320}