<commit_before>spawn = require("child_process").spawn
util = require("./util")

MODELPATH = __dirname + "/../model"

dict2vw = (dict) ->
  # Take a JavaScript key->value object, which I still call a "dict", and
  # express it as a string in the format VW expects.
  colonSeparated = ["#{key.replace(/[ ]/g, '_')}:#{value}" for key, value of dict]
  return colonSeparated.join().replace(/,/g, ' ')

featureString = (name, categories) ->
  namefix = JSON.stringify(name).replace(/[ ]/g, '_')
  catStrings = [category+' '+dict2vw(value) for category, value of categories]
  allCatString = catStrings.join().replace(/,/g, ' |')
  return "0 1 #{namefix}|#{allCatString}"

maximizePrediction = (modelName, vwInput, responder) ->
  proc = spawn('sh', ['vowpal-pipe.sh', "#{MODELPATH}/#{modelName}"])
  proc.stdin.setEncoding('utf8')
  proc.stdout.setEncoding('utf8')
  received = []
  proc.stdout.on 'data', (data) ->
    received.push(data)
  proc.stderr.on 'data', (data) ->
    console.log('stderr: '+data)
  proc.on 'exit', (code) ->
    if code
      responder.fail {
        errorCode: code
        model: modelName
        input: vwInput
      }
    else
      lines = received.join('').split('\n')
      choices = []
      for line in lines
        if line
          console.log(line)
          [scoreStr, name] = line.split(' ')
          name = JSON.parse(name.replace(/_/g, ' '))
          score = parseFloat(scoreStr)
          choices.push [name, score]
      
      if choices.length is 0
        responder.fail {
          error: 'no choices given'
          model: modelName
          input: vwInput
        }

      # Sort the choices in descending order by score.
      choices.sort (a, b) ->
        b[1] - a[1]
      responder.succeed {
        choices: choices
        best: choices[0][0]
        score: choices[0][1]<commit_msg>make output less chatty<commit_after>spawn = require("child_process").spawn
util = require("./util")

MODELPATH = __dirname + "/../model"

dict2vw = (dict) ->
  # Take a JavaScript key->value object, which I still call a "dict", and
  # express it as a string in the format VW expects.
  colonSeparated = ["#{key.replace(/[ ]/g, '_')}:#{value}" for key, value of dict]
  return colonSeparated.join().replace(/,/g, ' ')

featureString = (name, categories) ->
  namefix = JSON.stringify(name).replace(/[ ]/g, '_')
  catStrings = [category+' '+dict2vw(value) for category, value of categories]
  allCatString = catStrings.join().replace(/,/g, ' |')
  return "0 1 #{namefix}|#{allCatString}"

maximizePrediction = (modelName, vwInput, responder) ->
  proc = spawn('sh', ['vowpal-pipe.sh', "#{MODELPATH}/#{modelName}"])
  proc.stdin.setEncoding('utf8')
  proc.stdout.setEncoding('utf8')
  received = []
  proc.stdout.on 'data', (data) ->
    received.push(data)
  #proc.stderr.on 'data', (data) ->
  #  console.log('stderr: '+data)
  proc.on 'exit', (code) ->
    if code
      responder.fail {
        errorCode: code
        model: modelName
        input: vwInput
      }
    else
      lines = received.join('').split('\n')
      choices = []
      for line in lines
        if line
          [scoreStr, name] = line.split(' ')
          name = JSON.parse(name.replace(/_/g, ' '))
          score = parseFloat(scoreStr)
          choices.push [name, score]
      
      if choices.length is 0
        responder.fail {
          error: 'no choices given'
          model: modelName
          input: vwInput
        }

      # Sort the choices in descending order by score.
      choices.sort (a, b) ->
        b[1] - a[1]
      responder.succeed {
        choices: choices
        best: choices[0][0]
        score: choices[0][1]
{'fraction_non_alphanumeric': 0.15940054495912806, 'fraction_numerical': 0.007084468664850136, 'mean_word_length': 2.5230326295585415, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1737317', 'n_tokens_mistral': 1203, 'n_tokens_neox': 1204, 'n_words': 297}