<commit_before>"""
User-level analysis commands, typically for measuring or generating SheetViews.

Most of this file consists of commands for creating SheetViews, paired
with a template for how to construct plots from these SheetViews.

For instance, the implementation of Activity plots consists of the
update_activity() command plus the Activity PlotGroupTemplate.  The
update_activity() command reads the activity array of each Sheet and
makes a corresponding SheetView to put in the Sheet's sheet_views
dictionary, while the Activity PlotGroupTemplate specifies which
SheetViews should be plotted in which combination.  See the help for
PlotGroupTemplate for more information.

Some of the commands are ordinary Python functions, but the rest are
ParameterizedFunctions, which act like Python functions but support
Parameters with defaults, bounds, inheritance, etc.  These commands
are usually grouped together using inheritance so that they share a
set of parameters and some code, and only the bits that are specific
to that particular plot or analysis appear below.  See the
superclasses for the rest of the parameters and code.

$Id$
"""
__version__='$Revision$'

import Image,ImageDraw
    
import copy

from numpy.oldnumeric import array, maximum
from numpy import pi, sin, cos, nonzero, round, linspace

import param
from param.parameterized import ParameterizedFunction
from param.parameterized import ParamOverrides

import topo
from topo.base.cf import Projection
from topo.base.sheet import Sheet
from topo.sheet import GeneratorSheet
from topo.base.sheetview import SheetView
from topo.misc.distribution import Distribution, DistributionStatisticFn
from topo.misc.distribution import DSF_MaxValue, DSF_TopTwoValues, DSF_BimodalPeaks
from topo.misc.distribution import DSF_WeightedAverage, DSF_VonMisesFit, DSF_BimodalVonMisesFit
from topo.pattern import GaussiansCorner, Gaussian, RawRectangle, Composite, Constant
from topo.analysis.featureresponses import ReverseCorrelation
from topo.plotting.plotgroup import create_plotgroup, plotgroups

from topo.plotting.plotgroup import UnitMeasurementCommand,ProjectionSheetMeasurementCommand
from topo.analysis.featureresponses import Feature, PatternPresenter, MeasureResponseCommand
from topo.analysis.featureresponses import SinusoidalMeasureResponseCommand, PositionMeasurementCommand, SingleInputResponseCommand
from topo.base.patterngenerator import PatternGenerator
from topo.command import pattern_present

from topo.misc.patternfn import line


# Helper function for save_plotgroup
def _equivalent_for_plotgroup_update(p1,p2):
    """
    Helper function for save_plotgroup.
    
    Comparison operator for deciding whether make_plots(update==False) is
    safe for one plotgroup if the other has already been updated.
    
    Treats plotgroups as the same if the specified list of attributes
    (if present) match in both plotgroups.
    """

    attrs_to_check = ['pre_plot_hooks','keyname','sheet','x','y','projection','input_sheet','density','coords']

    for a in attrs_to_check:
        if hasattr(p1,a) or hasattr(p2,a):
            if not (hasattr(p1,a) and hasattr(p2,a) and getattr(p1,a)== getattr(p2,a)):
                return False

    return True



class save_plotgroup(ParameterizedFunction):
    """
    Convenience command for saving a set of plots to disk.  Examples:

      save_plotgroup("Activity")
      save_plotgroup("Orientation Preference")
      save_plotgroup("Projection",projection=topo.sim['V1'].projections('Afferent'))
                                  
    Some plotgroups accept optional parameters, which can be passed
    like projection above.
    """

    equivalence_fn = param.Callable(default=_equivalent_for_plotgroup_update,doc="""
        Function to call on plotgroups p1,p2 to determine if calling pre_plot_hooks
        on one of them is sufficient to update both plots.  Should return False
        unless the commands are exact equivalents, including all relevant parameters.""")

    use_cached_results = param.Boolean(default=False,doc="""
        If True, will use the equivalence_fn to determine cases where
        the pre_plot_hooks for a plotgroup can safely be skipped, to
        avoid lengthy redundant computation.  Should usually be
        False for safety, but can be enabled for e.g. batch mode
        runs using a related batch of plots.""")

    saver_params = param.Dict(default={},doc="""
        Optional parameters to pass to the underlying PlotFileSaver object.""")


    # Class variables to cache values from previous invocations
    previous_time=[-1]
    previous_plotgroups=[]

    def __call__(self,name,**params):
        p=ParamOverrides(self,params,allow_extra_keywords=True)

        plotgroup = copy.deepcopy(plotgroups[name])
    
        # JABALERT: Why does a Projection plot need a Sheet parameter?
        # CB: It shouldn't, of course, since we know the sheet when we have
        # the projection object - it's just leftover from when we passed the
        # names instead. There should be an ALERT already about this somewhere
        # in projectionpanel.py or plotgroup.py (both need to be changed).
        if 'projection' in params:
            setattr(plotgroup,'sheet',params['projection'].dest)
    
        plotgroup._set_name(name)
        
        # Specified parameters that aren't parameters of
        # save_plotgroup() are set on the plotgroup
        for n,v in p.extra_keywords().items():
            plotgroup.set_param(n,v)

        # Reset plot cache when time changes
        if (topo.sim.time() != self.previous_time[0]):
            del self.previous_time[:]
            del self.previous_plotgroups[:]
            self.previous_time.append(topo.sim.time())
            
        # Skip update step if equivalent to prior command at this sim time
        update=True
        if p.use_cached_results:
            for g in self.previous_plotgroups:
                if p.equivalence_fn(g,plotgroup):
                    update=False
                    break

        keywords=" ".join(["%s" % (v.name if isinstance(v,param.Parameterized) else str(v)) for n,v in p.extra_keywords().items()])
        plot_description="%s%s%s" % (plotgroup.name," " if keywords else "",keywords)
        if update:
            self.previous_plotgroups.append(plotgroup)
            self.debug("%s: Running pre_plot_hooks" % plot_description)
        else:
            self.message("%s: Using cached results from pre_plot_hooks" % plot_description)

        plotgroup.make_plots(update=update)
        plotgroup.filesaver.save_to_disk(**(p.saver_params))



def decode_feature(sheet, preference_map = "OrientationPreference", axis_bounds=(0.0,1.0), cyclic=True, weighted_average=True):
    """
    Estimate the value of a feature from the current activity pattern on a sheet.
    
    The specified preference_map should be measured before this
    function is called.
    
    If weighted_average is False, the feature value returned is the
    value of the preference_map at the maximally active location.
    
    If weighted_average is True, the feature value is estimated by
    weighting the preference_map by the current activity level, and
    averaging the result across all units in the sheet.  The
    axis_bounds specify the allowable range of the feature values in
    the preference_map.  If cyclic is true, a vector average is used;
    otherwise an arithmetic weighted average is used.
    
    For instance, if preference_map is OrientationPreference (a cyclic
    quantity), then the result will be the vector average of the
    activated orientations.  For an orientation map this value should
    be an estimate of the orientation present on the input.
    """
    d = Distribution(axis_bounds, cyclic)
    
    if not (preference_map in sheet.sheet_views):
        topo.sim.warning(preference_map + " should be measured before calling decode_feature.")
    else:
        map = sheet.sheet_views[preference_map]
        d.add(dict(zip(map.view()[0].ravel(), sheet.activity.ravel())))
    
    if weighted_average:
        return d.weighted_average()
    else:
        return d.max_value_bin()



def update_activity():
    """
    Make a map of neural activity available for each sheet, for use in template-based plots.
    
    This command simply asks each sheet for a copy of its activity
    matrix, and then makes it available for plotting.  Of course, for
    some sheets providing this information may be non-trivial, e.g. if
    they need to average over recent spiking activity.
    """
    for sheet in topo.sim.objects(Sheet).values():
        activity_copy = array(sheet.activity)
        new_view = SheetView((activity_copy,sheet.bounds),
                              sheet.name,sheet.precedence,topo.sim.time(),sheet.row_precedence)
        sheet.sheet_views['Activity']=new_view


pg = create_plotgroup(name='Activity',category='Basic',
             doc='Plot the activity for all Sheets.', auto_refresh=True,
             pre_plot_hooks=[update_activity], plot_immediately=True)
pg.add_plot('Activity',[('Strength','Activity')])


def update_rgb_activities():
    """
    Make available Red, Green, and Blue activity matrices for all appropriate sheets.
    """
    for sheet in topo.sim.objects(Sheet).values():<commit_msg>Fixed decode_feature after recent changes to Distribution, and added cropfn argument for doing tilt illusion experiments<commit_after>"""
User-level analysis commands, typically for measuring or generating SheetViews.

Most of this file consists of commands for creating SheetViews, paired
with a template for how to construct plots from these SheetViews.

For instance, the implementation of Activity plots consists of the
update_activity() command plus the Activity PlotGroupTemplate.  The
update_activity() command reads the activity array of each Sheet and
makes a corresponding SheetView to put in the Sheet's sheet_views
dictionary, while the Activity PlotGroupTemplate specifies which
SheetViews should be plotted in which combination.  See the help for
PlotGroupTemplate for more information.

Some of the commands are ordinary Python functions, but the rest are
ParameterizedFunctions, which act like Python functions but support
Parameters with defaults, bounds, inheritance, etc.  These commands
are usually grouped together using inheritance so that they share a
set of parameters and some code, and only the bits that are specific
to that particular plot or analysis appear below.  See the
superclasses for the rest of the parameters and code.

$Id$
"""
__version__='$Revision$'

import Image,ImageDraw
    
import copy

from numpy.oldnumeric import array, maximum
from numpy import pi, sin, cos, nonzero, round, linspace

import param
from param.parameterized import ParameterizedFunction
from param.parameterized import ParamOverrides

import topo
from topo.base.cf import Projection
from topo.base.sheet import Sheet
from topo.sheet import GeneratorSheet
from topo.base.sheetview import SheetView
from topo.misc.distribution import Distribution, DistributionStatisticFn
from topo.misc.distribution import DSF_MaxValue, DSF_TopTwoValues, DSF_BimodalPeaks
from topo.misc.distribution import DSF_WeightedAverage, DSF_VonMisesFit, DSF_BimodalVonMisesFit
from topo.pattern import GaussiansCorner, Gaussian, RawRectangle, Composite, Constant
from topo.analysis.featureresponses import ReverseCorrelation
from topo.plotting.plotgroup import create_plotgroup, plotgroups

from topo.plotting.plotgroup import UnitMeasurementCommand,ProjectionSheetMeasurementCommand
from topo.analysis.featureresponses import Feature, PatternPresenter, MeasureResponseCommand
from topo.analysis.featureresponses import SinusoidalMeasureResponseCommand, PositionMeasurementCommand, SingleInputResponseCommand
from topo.base.patterngenerator import PatternGenerator
from topo.command import pattern_present

from topo.misc.patternfn import line


# Helper function for save_plotgroup
def _equivalent_for_plotgroup_update(p1,p2):
    """
    Helper function for save_plotgroup.
    
    Comparison operator for deciding whether make_plots(update==False) is
    safe for one plotgroup if the other has already been updated.
    
    Treats plotgroups as the same if the specified list of attributes
    (if present) match in both plotgroups.
    """

    attrs_to_check = ['pre_plot_hooks','keyname','sheet','x','y','projection','input_sheet','density','coords']

    for a in attrs_to_check:
        if hasattr(p1,a) or hasattr(p2,a):
            if not (hasattr(p1,a) and hasattr(p2,a) and getattr(p1,a)== getattr(p2,a)):
                return False

    return True



class save_plotgroup(ParameterizedFunction):
    """
    Convenience command for saving a set of plots to disk.  Examples:

      save_plotgroup("Activity")
      save_plotgroup("Orientation Preference")
      save_plotgroup("Projection",projection=topo.sim['V1'].projections('Afferent'))
                                  
    Some plotgroups accept optional parameters, which can be passed
    like projection above.
    """

    equivalence_fn = param.Callable(default=_equivalent_for_plotgroup_update,doc="""
        Function to call on plotgroups p1,p2 to determine if calling pre_plot_hooks
        on one of them is sufficient to update both plots.  Should return False
        unless the commands are exact equivalents, including all relevant parameters.""")

    use_cached_results = param.Boolean(default=False,doc="""
        If True, will use the equivalence_fn to determine cases where
        the pre_plot_hooks for a plotgroup can safely be skipped, to
        avoid lengthy redundant computation.  Should usually be
        False for safety, but can be enabled for e.g. batch mode
        runs using a related batch of plots.""")

    saver_params = param.Dict(default={},doc="""
        Optional parameters to pass to the underlying PlotFileSaver object.""")


    # Class variables to cache values from previous invocations
    previous_time=[-1]
    previous_plotgroups=[]

    def __call__(self,name,**params):
        p=ParamOverrides(self,params,allow_extra_keywords=True)

        plotgroup = copy.deepcopy(plotgroups[name])
    
        # JABALERT: Why does a Projection plot need a Sheet parameter?
        # CB: It shouldn't, of course, since we know the sheet when we have
        # the projection object - it's just leftover from when we passed the
        # names instead. There should be an ALERT already about this somewhere
        # in projectionpanel.py or plotgroup.py (both need to be changed).
        if 'projection' in params:
            setattr(plotgroup,'sheet',params['projection'].dest)
    
        plotgroup._set_name(name)
        
        # Specified parameters that aren't parameters of
        # save_plotgroup() are set on the plotgroup
        for n,v in p.extra_keywords().items():
            plotgroup.set_param(n,v)

        # Reset plot cache when time changes
        if (topo.sim.time() != self.previous_time[0]):
            del self.previous_time[:]
            del self.previous_plotgroups[:]
            self.previous_time.append(topo.sim.time())
            
        # Skip update step if equivalent to prior command at this sim time
        update=True
        if p.use_cached_results:
            for g in self.previous_plotgroups:
                if p.equivalence_fn(g,plotgroup):
                    update=False
                    break

        keywords=" ".join(["%s" % (v.name if isinstance(v,param.Parameterized) else str(v)) for n,v in p.extra_keywords().items()])
        plot_description="%s%s%s" % (plotgroup.name," " if keywords else "",keywords)
        if update:
            self.previous_plotgroups.append(plotgroup)
            self.debug("%s: Running pre_plot_hooks" % plot_description)
        else:
            self.message("%s: Using cached results from pre_plot_hooks" % plot_description)

        plotgroup.make_plots(update=update)
        plotgroup.filesaver.save_to_disk(**(p.saver_params))



def decode_feature(sheet, preference_map = "OrientationPreference", axis_bounds=(0.0,1.0), cyclic=True, weighted_average=True, cropfn=lambda(x):x):
    """
    Estimate the value of a feature from the current activity pattern on a sheet.
    
    The specified preference_map should be measured before this
    function is called.
    
    If weighted_average is False, the feature value returned is the
    value of the preference_map at the maximally active location.
    
    If weighted_average is True, the feature value is estimated by
    weighting the preference_map by the current activity level, and
    averaging the result across all units in the sheet.  The
    axis_bounds specify the allowable range of the feature values in
    the preference_map.  If cyclic is true, a vector average is used;
    otherwise an arithmetic weighted average is used.
    
    For instance, if preference_map is OrientationPreference (a cyclic
    quantity), then the result will be the vector average of the
    activated orientations.  For an orientation map this value should
    be an estimate of the orientation present on the input.

    If desired, a cropfn can be supplied that will narrow the analysis
    down to a specific region of the array; this function will be
    applied to the preference_map and to the activity array before
    decoding.  Examples:

    Decode whole area:
 
       decode_feature(topo.sim["V1"])
    
    Decode left half only:

       r,c = topo.sim["V1"].activity.shape
       lefthalf  = lambda(x): x[:,0:c/2]
       righthalf = lambda(x): x[:,c/2:]

       decode_feature(topo.sim["V1"], cropfn=lefthalf)
    
    """

    d = Distribution(axis_bounds, cyclic)
    
    if not (preference_map in sheet.sheet_views):
        topo.sim.warning(preference_map + " should be measured before calling decode_feature.")
    else:
        v = sheet.sheet_views[preference_map]
        d.add(dict(zip(cropfn(v.view()[0]).ravel(), 
                       cropfn(sheet.activity).ravel())))
    
    res = DSF_WeightedAverage()(d) if weighted_average else DSF_MaxValue()(d)
    return res['']['preference']
    


def update_activity():
    """
    Make a map of neural activity available for each sheet, for use in template-based plots.
    
    This command simply asks each sheet for a copy of its activity
    matrix, and then makes it available for plotting.  Of course, for
    some sheets providing this information may be non-trivial, e.g. if
    they need to average over recent spiking activity.
    """
    for sheet in topo.sim.objects(Sheet).values():
        activity_copy = array(sheet.activity)
        new_view = SheetView((activity_copy,sheet.bounds),
                              sheet.name,sheet.precedence,topo.sim.time(),sheet.row_precedence)
        sheet.sheet_views['Activity']=new_view


pg = create_plotgroup(name='Activity',category='Basic',
             doc='Plot the activity for all Sheets.', auto_refresh=True,
             pre_plot_hooks=[update_activity], plot_immediately=True)
pg.add_plot('Activity',[('Strength','Activity')])


def update_rgb_activities():
    """
    Make available Red, Green, and Blue activity matrices for all appropriate sheets.
    """
    for sheet in topo.sim.objects(Sheet).values():
{'fraction_non_alphanumeric': 0.07205251460174704, 'fraction_numerical': 0.0021708792060784617, 'mean_word_length': 3.4995348837209304, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1518934', 'n_tokens_mistral': 5309, 'n_tokens_neox': 5044, 'n_words': 1862}