<commit_before># generate summary information for experience files
# see headings for outputs

fs = require 'fs'

if process.argv.length <= 2
  console.error 'usage: coffee experienceinfo <experience-files> ...'
  console.error 'Writes output to stdout as CSV'
  process.exit 0
  
headings = ['file','input','filters','nprofiles','profiles','ncodes','codes','npitch','nrhythm','nboth','nwildcards','wildcards','ninexact','errors','actions','actiontypes','nchannels','channels','state','npreconditions','preconditions','nexamples']

outrow = ( els ) ->
  out = ''
  for ei in [0 .. els.length-1]
    if ei > 0
      out += ','
    el = String els[ei]
    if el? and ((el.indexOf ',') >= 0 or (el.indexOf '"') >= 0)
      # escape
      out += '"'
      for ci in [0 .. el.length]
        c = el.charAt(ci)
        switch c
          when '"' then out += '""'
          else out += c
      out += '"'
    else
      out += el
  console.log out
  
outrow headings

wildcards = ['.','*','+','{']
actiontypes = 
  midi: 'data:text/x-midi-hex'
  osc: 'osc.udp:'
  text: 'data:text/plain'

# return info object
getinfo = ( exp ) ->
  info = {}
  # headings = ['file','input','filters','nprofiles','profiles','ncodes','nwildcards','wildcards','ninexact','errors','codes','nexamples','actions','nchannels','channels','state','npreconditions','preconditions']
  # input
  if exp.parameters?.midiInput?
    info.input = 'midi('+exp.parameters.midiInput+')'
  else 
    info.input = 'audio(' + (exp.parameters.audioInput ? 'default') + '/' + (exp.parameters.audioChannel ? 0) + ')'
  # filters
  info.filters = ''+
    (if (exp.parameters.frequencyRatio?) then 'frequencyRatio('+exp.parameters.frequencyRatio+') ' else '')+
    (if (exp.parameters.minFrequency?) then 'minFrequency('+exp.parameters.minFrequency+') ' else '')+
    (if (exp.parameters.maxFrequency?) then 'maxFrequency('+exp.parameters.maxFrequency+') ' else '')+
    (if (exp.parameters.minVelocity?) then 'minVelocity('+exp.parameters.minVelocity+') ' else '')+
    (if (exp.parameters.maxVelocity?) then 'maxVelocity('+exp.parameters.maxVelocity+') ' else '')
  # profiles
  if exp.projections?
    info.nprofiles = exp.projections.length
    info.profiles = (proj.id for proj in exp.projections).join ' '
  else
    info.nprofiles = 0
    info.profiles = ''
  # codes
  info.ncodes = info.nwildcards = info.ninexact = info.npreconditions = info.nchannels = info.npitch = info.nrhythm = info.nboth = 0
  info.codes = []
  info.actions = []
  info.errors = []
  info.channels = []
  info.preconditions = []
  info.wildcards = []
  info.actiontypes = []
  for marker in (exp.markers ? [])
    code = marker.code
    
    rhythm = (code.indexOf '/') >= 0
    pitch = /[a-gA-G]/.test code
    if rhythm
      if pitch
        info.nboth++
      else
        info.nrhythm++
    else if pitch
      info.npitch++
    
    wc = false

    for wildcard in wildcards
      if (code.indexOf wildcard) >= 0
        if (info.wildcards.indexOf wildcard)<0
          info.wildcards.push wildcard
        wc = true
    if wc
      info.nwildcards++      

    if marker.projection?
      code = marker.projection+':'+code
    else if marker.codeformat?
      # v.1
      code = marker.codeformat+':'+code
    info.ncodes++
    if (info.codes.indexOf code) < 0
      info.codes.push code

    if marker.inexact
      info.ninexact++
      info.errors.push marker.inexactError ? 0          
      <commit_msg>experienceinfo at start/end and fix wildcard . handling<commit_after># generate summary information for experience files
# see headings for outputs

fs = require 'fs'

if process.argv.length <= 2
  console.error 'usage: coffee experienceinfo <experience-files> ...'
  console.error 'Writes output to stdout as CSV'
  process.exit 0
  
headings = ['file','input','filters','nprofiles','profiles','ncodes','codes','npitch','nrhythm','nboth','atstart','atend','wholephrase','anywhere','nwildcards','wildcards','ninexact','errors','actions','actiontypes','nchannels','channels','state','npreconditions','preconditions','nexamples']

outrow = ( els ) ->
  out = ''
  for ei in [0 .. els.length-1]
    if ei > 0
      out += ','
    el = String els[ei]
    if el? and ((el.indexOf ',') >= 0 or (el.indexOf '"') >= 0)
      # escape
      out += '"'
      for ci in [0 .. el.length]
        c = el.charAt(ci)
        switch c
          when '"' then out += '""'
          else out += c
      out += '"'
    else
      out += el
  console.log out
  
outrow headings

wildcards = ['*','+','{','['] # . needs special treatment vs decimal point
actiontypes = 
  midi: 'data:text/x-midi-hex'
  osc: 'osc.udp:'
  text: 'data:text/plain'

# return info object
getinfo = ( exp ) ->
  info = {}
  # see headings 
  # input
  if exp.parameters?.midiInput?
    info.input = 'midi('+exp.parameters.midiInput+')'
  else 
    info.input = 'audio(' + (exp.parameters.audioInput ? 'default') + '/' + (exp.parameters.audioChannel ? 0) + ')'
  # filters
  info.filters = ''+
    (if (exp.parameters.frequencyRatio?) then 'frequencyRatio('+exp.parameters.frequencyRatio+') ' else '')+
    (if (exp.parameters.minFrequency?) then 'minFrequency('+exp.parameters.minFrequency+') ' else '')+
    (if (exp.parameters.maxFrequency?) then 'maxFrequency('+exp.parameters.maxFrequency+') ' else '')+
    (if (exp.parameters.minVelocity?) then 'minVelocity('+exp.parameters.minVelocity+') ' else '')+
    (if (exp.parameters.maxVelocity?) then 'maxVelocity('+exp.parameters.maxVelocity+') ' else '')
  # profiles
  if exp.projections?
    info.nprofiles = exp.projections.length
    info.profiles = (proj.id for proj in exp.projections).join ' '
  else
    info.nprofiles = 0
    info.profiles = ''
  # codes
  info.ncodes = info.nwildcards = info.ninexact = info.npreconditions = info.nchannels = info.npitch = info.nrhythm = info.nboth = 
    info.atstart = info.atend = info.wholephrase = info.anywhere = 0
  info.codes = []
  info.actions = []
  info.errors = []
  info.channels = []
  info.preconditions = []
  info.wildcards = []
  info.actiontypes = []
  for marker in (exp.markers ? [])
    code = marker.code
    
    atstart = marker.atStart ? (code.indexOf '^')==0
    atend = marker.atEnd ? (code.indexOf '$')>=0
    if atstart and atend
      info.wholephrase++
    else if atstart
      info.atstart++
    else if atend
      info.atend++
    else
      info.anywhere++

    rhythm = (code.indexOf '/') >= 0
    pitch = /[a-gA-G]/.test code
    if rhythm
      if pitch
        info.nboth++
      else
        info.nrhythm++
    else if pitch
      info.npitch++
    
    wc = false

    for wildcard in wildcards
      if (code.indexOf wildcard) >= 0
        if (info.wildcards.indexOf wildcard)<0
          info.wildcards.push wildcard
        wc = true
    # '.', not decimal point
    if /[^0-9]\.[^0-9]/.test code
      wc = true
      if (info.wildcards.indexOf '.')<0
        info.wildcards.push '.'
    if wc
      info.nwildcards++
     

    if marker.projection?
      code = marker.projection+':'+code
    else if marker.codeformat?
      # v.1
      code = marker.codeformat+':'+code
    info.ncodes++
    if (info.codes.indexOf code) < 0
      info.codes.push code

    if marker.inexact
      info.ninexact++
      info.errors.push marker.inexactError ? 0          
      
{'fraction_non_alphanumeric': 0.162428219852338, 'fraction_numerical': 0.005605687722176647, 'mean_word_length': 2.8398950131233596, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4489644', 'n_tokens_mistral': 2475, 'n_tokens_neox': 2383, 'n_words': 519}