<commit_before># !usr/bin/env python
# -*- coding: utf-8 -*-
#
# Licensed under a 3-clause BSD license.
#
# @Author: <NAME>
# @Date:   2017-08-21 17:11:22
# @Last modified by:   <NAME>
# @Last Modified time: 2017-11-08 10:52:16

from __future__ import print_function, division, absolute_import
from marvin import config
from marvin.utils.datamodel.dap import datamodel
from marvin.core.exceptions import MarvinUserWarning
from marvin.utils.datamodel.query.base import QueryParameter
from marvin.utils.datamodel.dap.base import Property
from marvin.utils.general import invalidArgs, isCallableWithArgs
from matplotlib.gridspec import GridSpec
from collections import defaultdict, OrderedDict
from astropy.visualization import hist as ahist
import matplotlib as mpl
import matplotlib.pyplot as plt
import mpl_scatter_density
import numpy as np
import scipy.stats as stats
import six
import seaborn as sns
import pandas as pd
import itertools


def compute_stats(data):
    ''' Compute some statistics given a data array

    Computes some basic statistics given a data array, excluding NaN values.
    Computes and returns the following Numpy statistics: mean, standard deviation,
    median, and the 10th, 25th, 75th, and 90th percentiles.

    Parameters:
        data (list|ndarray):
            A list or Numpy array of data

    Returns:
        A dictionary of statistics values

    '''
    stats = {'mean': np.nanmean(data), 'std': np.nanstd(data), 'median': np.nanmedian(data),
             'per10': np.nanpercentile(data, 10), 'per25': np.nanpercentile(data, 25),
             'per75': np.nanpercentile(data, 75), 'per90': np.nanpercentile(data, 90)}

    return stats


def _make_masked(data, mask=None):
    ''' Makes a masked array '''

    arr_data = data
    if not isinstance(data, np.ma.MaskedArray):
        # mask out NaN values if a mask not provided
        mask = mask if mask else np.isnan(data)
        # create array
        arr_data = np.ma.MaskedArray(data, mask=mask)

    return arr_data


def _create_figure(hist=None, hist_axes_visible=None, use_density=None):
    ''' Create a generic figure and axis '''

    # use a scatter density projection or not
    projection = 'scatter_density' if use_density else None

    # create the figure
    fig = plt.figure()
    ax_hist_x = None
    ax_hist_y = None

    # create axes with or without histogram
    if hist:
        if hist is True:
            gs = GridSpec(4, 4)
            ax_scat = fig.add_subplot(gs[1:4, 0:3], projection=projection)
            ax_hist_x = fig.add_subplot(gs[0, 0:3])
            ax_hist_y = fig.add_subplot(gs[1:4, 3])
        elif hist == 'x':
            gs = GridSpec(2, 1, height_ratios=[1, 2])
            ax_scat = fig.add_subplot(gs[1], projection=projection)
            ax_hist_x = fig.add_subplot(gs[0])
        elif hist == 'y':
            gs = GridSpec(1, 2, width_ratios=[2, 1])
            ax_scat = fig.add_subplot(gs[0], projection=projection)
            ax_hist_y = fig.add_subplot(gs[1])
    else:
        ax_scat = fig.add_subplot(1, 1, 1, projection=projection)

    # turn off histogram axes
    if ax_hist_x:
        plt.setp(ax_hist_x.get_xticklabels(), visible=hist_axes_visible)
    if ax_hist_y:
        plt.setp(ax_hist_y.get_yticklabels(), visible=hist_axes_visible)

    return fig, ax_scat, ax_hist_x, ax_hist_y


def _create_hist_title(data):
    ''' create a title for the histogram '''
    stats = compute_stats(data)
    hist_title = 'Stats: $\\mu={mean:.3f}, \\sigma={std:.3f}$'.format(**stats)
    return hist_title


def _get_dap_datamodel_property_label(quantity):
    ''' Format a DAP datamodel property string label '''
    return '{0} [{1}]'.format(quantity.to_string('latex'), quantity.unit.to_string('latex'))


def _get_axis_label(column, axis=''):
    ''' Create an axis label '''

    if isinstance(column, QueryParameter):
        if hasattr(column, 'property') and column.property:
            label = _get_dap_datamodel_property_label(column.property)
        else:
            label = column.display
    elif isinstance(column, Property):
        label = _get_dap_datamodel_property_label(column)
    elif isinstance(column, six.string_types):
        label = column
    else:
        # label = '{0} axis'.format(axis).strip()
        label = ''

    return label


def _set_options():
    ''' Set some default Matplotlib options '''
    mpl.rcParams['axes.axisbelow'] = True
    mpl.rcParams['grid.color'] = 'gray'
    mpl.rcParams['grid.linestyle'] = 'dashed'
    mpl.rcParams['grid.alpha'] = 0.8


def _set_limits(column, lim=None):
    ''' Set an axis limit '''
    if lim is not None:
        assert len(lim) == 2, 'range must be a list or tuple of 2'
    else:
        zscore = stats.zscore(column)
        # use percentile limits if the max zscore is > 50 sigma away from median
        if np.max(zscore) > 50.0:
            lim = [np.percentile(column, 0.5), np.percentile(column, 99.5)]
        else:
            pass
    return lim


def _check_input_data(coldim, col, data=None):
    ''' Check the input data '''

    # check data
    assert col is not None, 'Must provide an {0} column'.format(coldim)
<commit_msg>generalizing range limit computations<commit_after># !usr/bin/env python
# -*- coding: utf-8 -*-
#
# Licensed under a 3-clause BSD license.
#
# @Author: <NAME>
# @Date:   2017-08-21 17:11:22
# @Last modified by:   <NAME>
# @Last Modified time: 2017-11-08 11:24:09

from __future__ import print_function, division, absolute_import
from marvin import config
from marvin.utils.datamodel.dap import datamodel
from marvin.core.exceptions import MarvinUserWarning
from marvin.utils.datamodel.query.base import QueryParameter
from marvin.utils.datamodel.dap.base import Property
from marvin.utils.general import invalidArgs, isCallableWithArgs
from matplotlib.gridspec import GridSpec
from collections import defaultdict, OrderedDict
from astropy.visualization import hist as ahist
import matplotlib as mpl
import matplotlib.pyplot as plt
import mpl_scatter_density
import numpy as np
import scipy.stats as stats
import six
import seaborn as sns
import pandas as pd
import itertools


def compute_stats(data):
    ''' Compute some statistics given a data array

    Computes some basic statistics given a data array, excluding NaN values.
    Computes and returns the following Numpy statistics: mean, standard deviation,
    median, and the 10th, 25th, 75th, and 90th percentiles.

    Parameters:
        data (list|ndarray):
            A list or Numpy array of data

    Returns:
        A dictionary of statistics values

    '''
    stats = {'mean': np.nanmean(data), 'std': np.nanstd(data), 'median': np.nanmedian(data),
             'per10': np.nanpercentile(data, 10), 'per25': np.nanpercentile(data, 25),
             'per75': np.nanpercentile(data, 75), 'per90': np.nanpercentile(data, 90)}

    return stats


def _make_masked(data, mask=None):
    ''' Makes a masked array '''

    arr_data = data
    if not isinstance(data, np.ma.MaskedArray):
        # mask out NaN values if a mask not provided
        mask = mask if mask else np.isnan(data)
        # create array
        arr_data = np.ma.MaskedArray(data, mask=mask)

    return arr_data


def _create_figure(hist=None, hist_axes_visible=None, use_density=None):
    ''' Create a generic figure and axis '''

    # use a scatter density projection or not
    projection = 'scatter_density' if use_density else None

    # create the figure
    fig = plt.figure()
    ax_hist_x = None
    ax_hist_y = None

    # create axes with or without histogram
    if hist:
        if hist is True:
            gs = GridSpec(4, 4)
            ax_scat = fig.add_subplot(gs[1:4, 0:3], projection=projection)
            ax_hist_x = fig.add_subplot(gs[0, 0:3])
            ax_hist_y = fig.add_subplot(gs[1:4, 3])
        elif hist == 'x':
            gs = GridSpec(2, 1, height_ratios=[1, 2])
            ax_scat = fig.add_subplot(gs[1], projection=projection)
            ax_hist_x = fig.add_subplot(gs[0])
        elif hist == 'y':
            gs = GridSpec(1, 2, width_ratios=[2, 1])
            ax_scat = fig.add_subplot(gs[0], projection=projection)
            ax_hist_y = fig.add_subplot(gs[1])
    else:
        ax_scat = fig.add_subplot(1, 1, 1, projection=projection)

    # turn off histogram axes
    if ax_hist_x:
        plt.setp(ax_hist_x.get_xticklabels(), visible=hist_axes_visible)
    if ax_hist_y:
        plt.setp(ax_hist_y.get_yticklabels(), visible=hist_axes_visible)

    return fig, ax_scat, ax_hist_x, ax_hist_y


def _create_hist_title(data):
    ''' create a title for the histogram '''
    stats = compute_stats(data)
    hist_title = 'Stats: $\\mu={mean:.3f}, \\sigma={std:.3f}$'.format(**stats)
    return hist_title


def _get_dap_datamodel_property_label(quantity):
    ''' Format a DAP datamodel property string label '''
    return '{0} [{1}]'.format(quantity.to_string('latex'), quantity.unit.to_string('latex'))


def _get_axis_label(column, axis=''):
    ''' Create an axis label '''

    if isinstance(column, QueryParameter):
        if hasattr(column, 'property') and column.property:
            label = _get_dap_datamodel_property_label(column.property)
        else:
            label = column.display
    elif isinstance(column, Property):
        label = _get_dap_datamodel_property_label(column)
    elif isinstance(column, six.string_types):
        label = column
    else:
        # label = '{0} axis'.format(axis).strip()
        label = ''

    return label


def _set_options():
    ''' Set some default Matplotlib options '''
    mpl.rcParams['axes.axisbelow'] = True
    mpl.rcParams['grid.color'] = 'gray'
    mpl.rcParams['grid.linestyle'] = 'dashed'
    mpl.rcParams['grid.alpha'] = 0.8


def _set_limits(column, lim=None, sigma_cutoff=50, percent_clip=1):
    ''' Set an axis limit

    Determines whether to apply percentile clipping or not if any data
    has a zscore value above the sigma_cutoff value.  Applies percentile clipping
    centered around the mean.

    Parameters:
        column:
            The array of data to get limits of
        lim (list|tuple):
            A user provided range
        sigma_cutoff (int):
            The number of sigma away from the mean to cutoff
        percent_clip (int|tuple):
            The percent to clip off the data array.  Input values are taken as percentages.
            Can either be integer value (halved for lo,hi) or a tuple specifying lo,hi values.
            Default is 1%.

    Returns:
        A list of axis range values to use

    '''
    if lim is not None:
        assert len(lim) == 2, 'range must be a list or tuple of 2'
    else:
        # get percent clips
        if isinstance(percent_clip, (list, tuple)):
            lo, hi = percent_clip
        else:
            lo = percent_clip / 2.
            hi = 100 - lo

        zscore = stats.zscore(column)
        # use percentile limits if the max zscore is > 50 sigma away from mean/stdev
        if np.max(zscore) > sigma_cutoff:
            lim = [np.percentile(column, lo), np.percentile(column, hi)]
        else:
            pass
    return lim


def _check_input_data(coldim, col, data=None):
    ''' Check the input data '''

    # check data
    assert col is not None, 'Must provide an {0} column'.format(coldim)

{'fraction_non_alphanumeric': 0.10335712397568067, 'fraction_numerical': 0.01779892501541986, 'mean_word_length': 2.9423410906564778, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5813298', 'n_tokens_mistral': 3786, 'n_tokens_neox': 3520, 'n_words': 1060}