<commit_before>'''
General purpose  / miscellaneous functions.  Includes functions to approximate
continuous distributions with discrete ones, utility functions (and their
derivatives), manipulation of discrete distributions, and basic plotting tools.
'''

from __future__ import division     # Import Python 3.x division function
from __future__ import print_function
from builtins import str
from builtins import range
from builtins import object
import functools

import numpy as np                  # Python's numeric library, abbreviated "np"
import math
# try:
#     import matplotlib.pyplot as plt                 # Python's plotting library
# except ImportError:
#     import sys
#     exception_type, value, traceback = sys.exc_info()
#     raise ImportError('HARK must be used in a graphical environment.', exception_type, value, traceback)
from scipy.interpolate import interp1d
import warnings


def memoize(obj):
   '''
   A decorator to (potentially) make functions more efficient.

   With this decorator, functions will "remember" if they have been evaluated with given inputs
   before.  If they have, they will "remember" the outputs that have already been calculated
   for those inputs, rather than calculating them again.
   '''
   cache = obj._cache = {}

   @functools.wraps(obj)
   def memoizer(*args, **kwargs):
       key = str(args) + str(kwargs)
       if key not in cache:
           cache[key] = obj(*args, **kwargs)
       return cache[key]<commit_msg>don't need math in utilities any more<commit_after>'''
General purpose  / miscellaneous functions.  Includes functions to approximate
continuous distributions with discrete ones, utility functions (and their
derivatives), manipulation of discrete distributions, and basic plotting tools.
'''

from __future__ import division     # Import Python 3.x division function
from __future__ import print_function
from builtins import str
from builtins import range
from builtins import object
import functools

import numpy as np                  # Python's numeric library, abbreviated "np"

# try:
#     import matplotlib.pyplot as plt                 # Python's plotting library
# except ImportError:
#     import sys
#     exception_type, value, traceback = sys.exc_info()
#     raise ImportError('HARK must be used in a graphical environment.', exception_type, value, traceback)
from scipy.interpolate import interp1d
import warnings


def memoize(obj):
   '''
   A decorator to (potentially) make functions more efficient.

   With this decorator, functions will "remember" if they have been evaluated with given inputs
   before.  If they have, they will "remember" the outputs that have already been calculated
   for those inputs, rather than calculating them again.
   '''
   cache = obj._cache = {}

   @functools.wraps(obj)
   def memoizer(*args, **kwargs):
       key = str(args) + str(kwargs)
       if key not in cache:
           cache[key] = obj(*args, **kwargs)
       return cache[key]
{'fraction_non_alphanumeric': 0.07082913729439409, 'fraction_numerical': 0.001342732460557234, 'mean_word_length': 3.65625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2709269', 'n_tokens_mistral': 791, 'n_tokens_neox': 750, 'n_words': 331}