<commit_before># A set of IPython helper methods

from glimpse import util
import numpy
np = numpy
from glimpse import core
import os
#import odd

load = util.Load
store = util.Store

def stats(x):
  return dict((k, getattr(x, k)()) for k in ('min', 'max', 'mean', 'std'))

def array3D(x):
  """Reshape array to have three dimensions, collapsing remaining dimensions."""
  return x.reshape((-1,) + x.shape[-2:])

def array2D(x):
  """Reshape array to have two dimensions, collapsing remaining dimensions."""
  return x.reshape((-1, x.shape[-1]))

WITH_PLOT = ('DISPLAY' in os.environ)
if WITH_PLOT:
  import glimpse.util.plot as gplot
  from matplotlib import pyplot
  def hist(x, **args):
    from matplotlib import pylab
    default_args = dict(bins = 100, hold = False)
    return pylab.hist(x.flat, **dict(default_args.items() + args.items()))

  def show(x, **args):
    if hasattr(x, 'show'):
      x.show()
    elif isinstance(x, np.ndarray):
      from matplotlib import pyplot
      pyplot.clf()
      if len(x.shape) == 1:
        pyplot.plot(x, **args)
      elif len(x.shape) == 2:
        gplot.Show3DArray(x, **args)
      else:
        gplot.Show3DArray(x, **args)
      pyplot.draw()
    elif isinstance(x, list) or isinstance(x, tuple):
      if isinstance(x[0], np.ndarray):
        gplot.Show2DArrayList(x, **args)
      else:
        raise ValueError("Can't display list for objects of type: %s" % type(x))
    else:
      raise ValueError("Can't display object of type: %s" % type(x))
<commit_msg>changed ipython.stats() to return an object<commit_after># A set of IPython helper methods

from glimpse import util
import numpy
np = numpy
from glimpse import core
import os
#import odd

load = util.Load
store = util.Store

class stats(object):
  def __init__(self, x):
    self.min = x.min()
    self.max = x.max()
    self.mean = x.mean()
    self.std = x.std()
  def __str__(self):
    return "min = %s, max = %s, mean = %s, std = %s" % (self.min, self.max, self.mean, self.std)
  def __repr__(self):
    return str(self)

def array3D(x):
  """Reshape array to have three dimensions, collapsing remaining dimensions."""
  return x.reshape((-1,) + x.shape[-2:])

def array2D(x):
  """Reshape array to have two dimensions, collapsing remaining dimensions."""
  return x.reshape((-1, x.shape[-1]))

WITH_PLOT = ('DISPLAY' in os.environ)
if WITH_PLOT:
  import glimpse.util.plot as gplot
  from matplotlib import pyplot
  def hist(x, **args):
    from matplotlib import pylab
    default_args = dict(bins = 100, hold = False)
    return pylab.hist(x.flat, **dict(default_args.items() + args.items()))

  def show(x, **args):
    if hasattr(x, 'show'):
      x.show()
    elif isinstance(x, np.ndarray):
      from matplotlib import pyplot
      if len(x.shape) == 1:
        pyplot.plot(x, **args)
      elif len(x.shape) == 2:
        gplot.Show3DArray(x, **args)
      else:
        gplot.Show3DArray(x, **args)
      pyplot.draw()
    elif isinstance(x, list) or isinstance(x, tuple):
      if isinstance(x[0], np.ndarray):
        gplot.Show2DArrayList(x, **args)
      else:
        raise ValueError("Can't display list for objects of type: %s" % type(x))
    else:
      raise ValueError("Can't display object of type: %s" % type(x))

{'fraction_non_alphanumeric': 0.1436023320036821, 'fraction_numerical': 0.009205277692543725, 'mean_word_length': 3.206451612903226, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3260824', 'n_tokens_mistral': 1143, 'n_tokens_neox': 1116, 'n_words': 339}