<jupyter_start><jupyter_text>Monitor batchesThis notebook allow you to monitor the spikeforest processing batches<jupyter_code># Only run this cell if you are running this on a hosted runtime that does not have these packages installed
# %%capture is used to suppress the output... this should take up to a minute to complete
%%capture
!pip install spikeforest
# Import the python packages -- autoreload is used for development purposes
%load_ext autoreload
%autoreload 2

import spikeforest as sf
import spikewidgets as sw
import vdomr as vd
import json

## Configure readonly access to kbucket
sf.kbucketConfigRemote(name='spikeforest1-readonly')

## Load the SpikeForest data
#SF=sf.SFData()
#SF.loadRecordings(key=dict(name='spikeforest_bionet_recordings'))
#SF.loadRecordings(key=dict(name='spikeforest_magland_synth_recordings'))
#SF.loadProcessingBatch(key=dict(batch_name='summarize_recordings_bionet',name='job_results'))
#SF.loadProcessingBatch(key=dict(batch_name='summarize_recordings_magland_synth',name='job_results'))
from kbucket import client as kb
from pairio import client as pa

class BatchJob():
  def __init__(self,object):
    self._object=object
  def command(self):
    return self._object['command']
  def label(self):
    return self._object['label']
  def object(self):
    return self._object
  def getStatus(self):
    val=pa.get(key=self._object)
    if not val:
      return 'not-started'
    if val.startswith('in-process-'):
      return 'in-process'
    elif val.startswith('error-'):
      return 'error'
    result=kb.loadObject(key=self._object)
    if not result:
      return 'result-not-found'
    return 'finished'

class Batch():
  def __init__(self,batch_name):
    self._batch_name=batch_name
    self._batch=kb.loadObject(key=dict(batch_name=batch_name))
  def numJobs(self):
    return len(self._batch['jobs'])
  def job(self,index):
    return BatchJob(self._batch['jobs'][index])
class BatchViewTable(vd.Component):
  def __init__(self):
    vd.Component.__init__(self)
    self._batch=None
  def setBatch(self,batch):
    self._batch=batch
    self.refresh()
  def render(self):
    if not self._batch:
      return vd.span('...')
    
    rows=[]
    header_row=vd.tr(
        vd.th('Job'),
        vd.th('Command'),
        vd.th('Status')
    )
    rows.append(header_row)
    for i in range(self._batch.numJobs()):
      job=self._batch.job(i)
      
      try:
        status=job.getStatus()
      except:
        status='not found'
      
      row=vd.tr(
          vd.td(job.label()),
          vd.td(job.command()),
          vd.td(status)
      )
      rows.append(row)
    table=vd.table(rows,class_='table')
    table2=vd.div(table,style=dict(width='700px',height='400px',overflow='auto'))
    return table2

class BatchView(vd.Component):
  def __init__(self,batch_name):
    vd.devel.loadBootstrap()
    vd.Component.__init__(self)
    self._batch_name=batch_name
    self._batch_view_table=BatchViewTable()
  def _reload_batch(self):
    batch=Batch(batch_name=self._batch_name)
    self._batch_view_table.setBatch(batch)
    self.refresh()
  def reloadBatch(self):
    self._reload_batch()
  def render(self):
    heading=vd.div(
        vd.h3('Batch: '+self._batch_name)
    )
    button_bar=vd.div(
        vd.components.Button('Refresh',onclick=self._reload_batch,class_='btn')
    )
    table=self._batch_view_table
    div=vd.div(
        heading,
        button_bar,
        table
    )
    return div
BV=BatchView(batch_name='ms4_mearec_tetrode')
display(BV)
BV.reloadBatch()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12105116019010344, 'fraction_numerical': 0.0033547665641599105, 'mean_word_length': 3.841677943166441, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '629411', 'license': 'Apache-2.0', 'n_tokens_mistral': 1218, 'n_tokens_neox': 1174, 'n_words': 236, 'path': 'notebooks/monitor_batches.ipynb', 'repo_name': 'magland/spikeforest_batch_run', 'size': '3577'}