<commit_before>#!/usr/bin/env python

import os
import time
import sys
import glob
import json
import logging


if (sys.platform == 'darwin') and (os.name == 'posix'):
  # this is the only one that seems to work on Mac OSX with animation...
  import matplotlib
  matplotlib.use('TkAgg')

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.ticker as mplticker
import matplotlib.animation as animation

import selutil

from IPython import embed
import pdb

# The directory to look for json files.
TMPDIR = '/tmp/cal-dvmdostem'
YRTMPDIR = '/tmp/year-cal-dvmdostem'

# some logging stuff
LOG_FORMAT = '%(levelname)-7s %(name)-8s %(message)s'
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)


class ExpandingWindow(object):
  '''An set of expanding window plots that all share the x axis.
  '''

  def __init__(self, traceslist, figtitle="Expanding Window Plot",
      rows=2, cols=1):

    logging.debug("Ctor for Expanding Window plot...")

    self.traces = traceslist

    self.fig, self.axes = plt.subplots(rows, cols, sharex='all')
    self.fig.suptitle(figtitle)
    
    plt.xlabel("Years")
    
    x = np.arange(0)
    y = x.copy() * np.nan
  
    logging.info("Setting up empty x,y data for every trace...")
    for trace in self.traces:
      ax = self.axes[ trace['axesnum'] ]
      trace['artists'] = ax.plot(x, y, label=trace['jsontag'])
    self.relim_autoscale_draw()
    self.grid_and_legend()
    
    self.loademupskis()
  
    logging.info("Done creating an expanding window plot object...")

  def init(self):
    logging.info("Init function for animation")

    return [trace['artists'][0] for trace in self.traces]




  def report_view_and_data_lims(self):
    logging.info("{0:-^56s}".format("Report Data and View Limits"))
    for i, ax in enumerate(self.axes):
      (dx0,dy0),(dx1,dy1) = ax.dataLim.get_points()
      (vx0,vy0),(vx1,vy1) = ax.viewLim.get_points()
      logging.info("{0:>10s} {1:>10.3f} {2:>10.3f} {3:>10.3f} {4:>10.3f}".format('Axes%i X:'%i, dx0,dx1, vx0,vx1))
      logging.info("{0:>10s} {1:>10.3f} {2:>10.3f} {3:>10.3f} {4:>10.3f}".format('Axes%i Y:'%i, dy0,dy1, vy0, vy1))



  def loademupskis(self):
    logging.info("LOAD 'EM UPSKIS!")
    
    files = sorted( glob.glob('%s/*.json' % YRTMPDIR) )
    logging.info("%i json files in %s" % (len(files), YRTMPDIR) )

    # create an x range big enough for every possible file...
    if len(files) == 0:
      x = np.arange(0)
    else:
      end = int( os.path.basename(files[-1])[0:4] )
      x = np.arange(0, end + 1 , 1) # <-- make range inclusive!
    
    # for each trace, create a tmp y container the same size as x
    for trace in self.traces:
      trace['tmpy'] = x.copy() * np.nan
    
    # ----- READ EVERY FILE --------
    logging.info("Read every file and load into trace['tmpy'] container")
    for file in files:
      # try reading the file
      try:
        with open(file) as f:
          fdata = json.load(f)
      except IOError as e:
        logging.error(e)
      
      idx = int(os.path.basename(file)[0:4])

      for trace in self.traces:
        # set the trace's tmpy[idx] to file's data
        if 'pft' in trace.keys():
          pftdata = fdata[trace['pft']]
          trace['tmpy'][idx] = pftdata[trace['jsontag']]
        else:
          trace['tmpy'][idx] = fdata[trace['jsontag']]

      
    # ----- UPDATE EVERY TRACE --------
    logging.info("Load tmp data for every trace to trace's line")
    for trace in self.traces:
      # find the line with the right label
      for line in self.axes[trace['axesnum']].lines:
        # set the line's data to x, and the trace's tmp data
        if line.get_label() == trace['jsontag']:
          line.set_data(x, trace['tmpy'])
        else:
          pass # wrong line...
  
    # clean up temproary storage
    for trace in self.traces:
      del trace['tmpy']


    # ----- RELIMIT and SCALE
    self.relim_autoscale_draw()
    self.grid_and_legend()
    self.report_view_and_data_lims()
  
    logging.info("Done loading 'em upskis.")




  def update(self, frame):
    logging.info("Frame %7i" % frame)
    
    # get current data and view limits
    for i, ax in enumerate(self.axes):
      (dx0,dy0),(dx1,dy1) = ax.dataLim.get_points()
      (vx0,vy0),(vx1,vy1) = ax.viewLim.get_points()
    

    files = sorted( glob.glob('%s/*.json' % YRTMPDIR) )
    logging.info("%i json files in %s" % (len(files), YRTMPDIR) )
    
    if int( os.path.basename(files[-1])[0:4] ) > dx1:
      pass
      # the largest year file will not fit in the data container...
      # must resize, then try again
        
    else:
      pass
      # start at the last file and put it in the
    

    # create an x range big enough for every possible file...
    if len(files) == 0:
      x = np.arange(0)
    else:
      end = int( os.path.basename(files[-1])[0:4] )
      x = np.arange(0, end + 1 , 1) # <-- make range inclusive!

    self.report_view_and_data_lims()
    
    self.loademupskis()

    return [trace['artists'][0] for trace in self.traces]


  def relim_autoscale_draw(self):
    logging.debug("Relimit axes, autoscale axes.")
    for ax in self.axes:
      ax.relim()
      ax.autoscale()
    logging.info("Redraw plot")
    plt.draw()

  def grid_and_legend(self):
    logging.debug("Turn on grid and legend.")
    for ax in self.axes:
      ax.grid(True) # <-- w/o parameter, this toggles!!
      ax.legend(prop={'size':10.0})

<commit_msg>Zoom close to workign for expanding window.<commit_after>#!/usr/bin/env python

import os
import time
import sys
import glob
import json
import logging


if (sys.platform == 'darwin') and (os.name == 'posix'):
  # this is the only one that seems to work on Mac OSX with animation...
  import matplotlib
  matplotlib.use('TkAgg')

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.ticker as mplticker
import matplotlib.animation as animation

import selutil

from IPython import embed
import pdb

# The directory to look for json files.
TMPDIR = '/tmp/cal-dvmdostem'
YRTMPDIR = '/tmp/year-cal-dvmdostem'

# some logging stuff
LOG_FORMAT = '%(levelname)-7s %(name)-8s %(message)s'
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)


class ExpandingWindow(object):
  '''An set of expanding window plots that all share the x axis.
  '''

  def __init__(self, traceslist, figtitle="Expanding Window Plot",
      rows=2, cols=1):

    logging.debug("Ctor for Expanding Window plot...")

    self.traces = traceslist

    self.fig, self.axes = plt.subplots(rows, cols, sharex='all')
    self.fig.suptitle(figtitle)
    
    self.fig.canvas.mpl_connect('key_press_event', self.key_press_event)

    plt.xlabel("Years")
    
    x = np.arange(0)
    y = x.copy() * np.nan
  
    logging.info("Setting up empty x,y data for every trace...")
    for trace in self.traces:
      ax = self.axes[ trace['axesnum'] ]
      trace['artists'] = ax.plot(x, y, label=trace['jsontag'])
    self.relim_autoscale_draw()
    self.grid_and_legend()
    
    self.loademupskis(relim=True, autoscale=True)
  
    logging.info("Done creating an expanding window plot object...")


  def init(self):
    logging.info("Init function for animation")

    return [trace['artists'][0] for trace in self.traces]

  def key_press_event(self, event):
    print('you pressed', event.key, event.xdata, event.ydata)
    if event.key == 'ctrl+r':
      self.loademupskis(relim=True, autoscale=True)


  def report_view_and_data_lims(self):
    logging.info("{0:>10s} {1:>10s} {2:>10s} {3:>10s} {4:>10s}".format('---','d0', 'd1', 'v0', 'v1'))
    for i, ax in enumerate(self.axes):
      (dx0,dy0),(dx1,dy1) = ax.dataLim.get_points()
      (vx0,vy0),(vx1,vy1) = ax.viewLim.get_points()
      logging.info("{0:>10s} {1:>10.3f} {2:>10.3f} {3:>10.3f} {4:>10.3f}".format('Axes%i X:'%i, dx0,dx1, vx0,vx1))
      logging.info("{0:>10s} {1:>10.3f} {2:>10.3f} {3:>10.3f} {4:>10.3f}".format('Axes%i Y:'%i, dy0,dy1, vy0, vy1))



  def loademupskis(self, relim, autoscale):
    logging.info("LOAD 'EM UPSKIS!")
    
    files = sorted( glob.glob('%s/*.json' % YRTMPDIR) )
    logging.info("%i json files in %s" % (len(files), YRTMPDIR) )

    # create an x range big enough for every possible file...
    if len(files) == 0:
      x = np.arange(0)
    else:
      end = int( os.path.basename(files[-1])[0:4] )
      x = np.arange(0, end + 1 , 1) # <-- make range inclusive!
    
    # for each trace, create a tmp y container the same size as x
    for trace in self.traces:
      trace['tmpy'] = x.copy() * np.nan
    
    # ----- READ EVERY FILE --------
    logging.info("Read every file and load into trace['tmpy'] container")
    for file in files:
      # try reading the file
      try:
        with open(file) as f:
          fdata = json.load(f)
      except IOError as e:
        logging.error(e)
      
      idx = int(os.path.basename(file)[0:4])

      for trace in self.traces:
        # set the trace's tmpy[idx] to file's data
        if 'pft' in trace.keys():
          pftdata = fdata[trace['pft']]
          trace['tmpy'][idx] = pftdata[trace['jsontag']]
        else:
          trace['tmpy'][idx] = fdata[trace['jsontag']]

      
    # ----- UPDATE EVERY TRACE --------
    logging.info("Load tmp data for every trace to trace's line")
    for trace in self.traces:
      # find the line with the right label
      for line in self.axes[trace['axesnum']].lines:
        # set the line's data to x, and the trace's tmp data
        if line.get_label() == trace['jsontag']:
          line.set_data(x, trace['tmpy'])
        else:
          pass # wrong line...
  
    # clean up temproary storage
    for trace in self.traces:
      del trace['tmpy']


    # ----- RELIMIT and SCALE
    if relim:
      logging.info("Recomputing data limits based on artist data")
      for ax in self.axes:
        ax.relim()
    if autoscale:
      for ax in self.axes:
        ax.autoscale(enable=True, axis='both', tight=False)
      logging.info("Force draw after autoscale")
      plt.draw()

    #self.grid_and_legend()

    self.report_view_and_data_lims()
  
    logging.info("Done loading 'em upskis.")




  def update(self, frame):
    logging.info("Frame %7i" % frame)
    
    # get current data and view limits
    for i, ax in enumerate(self.axes):
      (dx0,dy0),(dx1,dy1) = ax.dataLim.get_points()
      (vx0,vy0),(vx1,vy1) = ax.viewLim.get_points()
    
    files = sorted( glob.glob('%s/*.json' % YRTMPDIR) )
    logging.info("%i json files in %s" % (len(files), YRTMPDIR) )

    self.report_view_and_data_lims()

    if vx0 > dx0 or vx1 < dx1 or vy0 > dy0 or vy1 < dy1:
      logging.warn("ALERT: The user must be zoomed in...")
      logging.info("Update the artists, and recompute the data limits, but don't touch the view")
      self.loademupskis(relim=True, autoscale=False)
      logging.info("Return empty artist list. Don't re-draw anything while zoomed in.")
      return []
      
    else:
    
      self.loademupskis(relim=True, autoscale=True)
      logging.info("Return all the trace artists to redraw")
      return [trace['artists'][0] for trace in self.traces]




      # need to update data, but don't re-scale!

#    if len(files) == 0:
#      logging.info("No files. Clear the lines and set them to some min size.")
#      for ax in self.axes:
#        for line in axes.lines:
#          line.set_data(np.arange(0,100), np.empty(100)*np.nan)
#      self.relim_autoscale_draw()
#      self.grid_and_legend()
#      return [trace['artists'][0] for trace in self.traces]
  
  
    
#    if int( os.path.basename(files[-1])[0:4] ) > dx1:
#      logging.info("The largest year won't fit in the data extents...")
#      self.loademupskis()
    
#    else:
#      logging.info("The largest year WILL fit in the data extents...")
#      logging.info("Read every file and load into the right trace's data container")
#      for file in files:
#        # try reading the file
#        try:
#          with open(file) as f:
#            fdata = json.load(f)
#
#          idx = int(os.path.basename(file)[0:4])
#
#          for trace in self.traces:
#            for ax in self.axes:
#              for line in ax.lines:
#                if line.get_label() == trace['jsontag']:
#                  x, y = line.get_data()
#                  if 'pft' in trace.keys():
#                    pftdata = fdata[trace['pft']]
#                    y[idx] = pftdata[trace['jsontag']]
#                  else:
#                    ydata = fdata[trace['jsontag']]
#
#
#        except IOError as e:
#          logging.error(e)
        
  
      


  def relim_autoscale_draw(self):
    logging.debug("Relimit axes, autoscale axes.")
    for ax in self.axes:
      ax.relim()
      ax.autoscale(enable=True, axis='both', tight=False)
    logging.info("Redraw plot")
    try:
      plt.draw()
    except Exception as e:
      logging.error(e)

  def grid_and_legend(self):
    logging.debug("Turn on grid and legend.")
    for ax in self.axes:
      ax.grid(True) # <-- w/o parameter, this toggles!!
      ax.legend(prop={'size':10.0})


{'fraction_non_alphanumeric': 0.14591291061879297, 'fraction_numerical': 0.01703590527119939, 'mean_word_length': 2.5914951989026065, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2112143', 'n_tokens_mistral': 4806, 'n_tokens_neox': 4586, 'n_words': 1208}