<commit_before>#!/usr/bin/env python

"""
This module contains the :class:`Analysis` class, which is used for creating
optimized, repeatable data processing workflows. An analysis can be created
from a function or any other callable object. Dependent analyses can then be
created using the :meth:`Analysis.then` method. Each function must accept a
``data`` argument, which is a :class:`dict` of data to be persisted between
analyses. Modifications made to ``data`` in the scope of one analysis will be
propogated to all dependent analyses.

When :meth:`Analysis.run` is invoked, the analysis function runs, followed by
each of dependent analysis created with :meth:`Analysis.then`. These in turn
invoke their own dependent analyses, allowing a hierarchy to be created. The
result of each analysis will be cached to disk along with a "fingerprint"
describing the source of the analysis function at the time it was invoked. If
you run the same analysis twice without modifying the code, the cached version
out of the ``data`` will be used for its dependents. Thus you experiment with
a dependent analysis without constantly recomputing the results of its parent.
"""

import bz2
from copy import deepcopy
import hashlib
import inspect
import os

try:
    import cPickle as pickle
except ImportError: # pragma: no cover
    import pickle

from agate.utils import memoize

class Analysis(object):
    """
    An Analysis is a function whose code configuration and output can be
    serialized to disk. When it is invoked again, if it's code has not changed
    the serialized output will be used rather than executing the code again.

    Implements a promise-like API so that Analyses can depend on one another.<commit_msg>Add warning about non-recursive parsing in Analysis. Closes #188.<commit_after>#!/usr/bin/env python

"""
This module contains the :class:`Analysis` class, which is used for creating
optimized, repeatable data processing workflows. An analysis can be created
from a function or any other callable object. Dependent analyses can then be
created using the :meth:`Analysis.then` method. Each function must accept a
``data`` argument, which is a :class:`dict` of data to be persisted between
analyses. Modifications made to ``data`` in the scope of one analysis will be
propogated to all dependent analyses.

When :meth:`Analysis.run` is invoked, the analysis function runs, followed by
each of dependent analysis created with :meth:`Analysis.then`. These in turn
invoke their own dependent analyses, allowing a hierarchy to be created. The
result of each analysis will be cached to disk along with a "fingerprint"
describing the source of the analysis function at the time it was invoked. If
you run the same analysis twice without modifying the code, the cached version
out of the ``data`` will be used for its dependents. Thus you experiment with
a dependent analysis without constantly recomputing the results of its parent.

.. warning::

    The fingerprint which is generated for each analysis function is **not**
    recursive, which is to say, it does not include the source of functions
    which are invoked by that function. If you modify the source of a function
    invoked by the analysis function, you will need to ensure that the analysis
    is manually refreshed by passing ``refresh=True`` to :meth:`Analysis.run`.
"""

import bz2
from copy import deepcopy
import hashlib
import inspect
import os

try:
    import cPickle as pickle
except ImportError: # pragma: no cover
    import pickle

from agate.utils import memoize

class Analysis(object):
    """
    An Analysis is a function whose code configuration and output can be
    serialized to disk. When it is invoked again, if it's code has not changed
    the serialized output will be used rather than executing the code again.

    Implements a promise-like API so that Analyses can depend on one another.
{'fraction_non_alphanumeric': 0.050538737814263726, 'fraction_numerical': 0.0012827090815802976, 'mean_word_length': 4.3853591160221, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4072208', 'n_tokens_mistral': 969, 'n_tokens_neox': 924, 'n_words': 573}