<commit_before>/*!
* mingy
* Copyright(c) 2011 <NAME>
* MIT Licensed
*/

var util = require('util')
  , Command = require('./command').Command

exports.Parser = function Parser(commands) {

  this.commands = commands || {}
  this.validators = {}
  this.env = {}
}

exports.Parser.prototype = {

  setEnv: function(property, value) {
    this.env[property] = value
  },

  addCommand: function(name) {

    // throw error if name already taken
    var command = new Command('name')
    this.commands[name] = command
    return command
  },

  addValidator: function(name, logic) {
    this.validators[name] = logic
  },

  parse: function(input, stream) {

    input = this.cleanInput(input)
    var lexemes = input.split(' ')
    return this.parseLexemes(lexemes, stream)
  },

  cleanInput: function(input) {

    // server shell sends extra junk
    input = input.replace("\r\n", "\n")

    // remove trailing newline
    if (input.slice(-1) == "\n") {
      input = input.slice(0, input.length - 1)
    }

    // remove redundant spaces
    while (input.indexOf('  ') != -1) {
      input = input.replace('  ', ' ')
    }

    return input
  },

  parseLexemes: function(lexemes, stream) {

    var output = ''

    // cycle through commands looking for syntax match
    for (var index in this.commands) {

      var command = this.commands[index]

      // we clone lexemes because if the last syntax lexeme has a wildcard the
      // submitted lexeme corresponding to the last syntax lexeme ends up
      // getting subsequent submitted lexemes added to it
      if (command.try(this.validators, this.env, this.clone(lexemes), true)) {

        // command condition jazz goes here
        if (1) {

          result = command.try(this.validators, this.env, lexemes, false, stream)

          if (result) {
            return output + result
          }
        }
        else if(output) {

          return output
        }
      }
    }
  },

  clone: function(obj) {
    var newObj = (obj instanceof Array) ? [] : {};
    for (i in obj) {
      if (i == 'clone') continue;
      if (obj[i] && typeof obj[i] == "object") {
        newObj[i] = this.clone(obj[i]);
      } else newObj[i] = obj[i]
    } return newObj;
  }
}<commit_msg>Added function to convert a URL to lexemes.<commit_after>/*!
* mingy
* Copyright(c) 2011 <NAME>
* MIT Licensed
*/

var util = require('util')
  , Command = require('./command').Command

exports.Parser = function Parser(commands) {

  this.commands = commands || {}
  this.validators = {}
  this.env = {}
}

exports.Parser.prototype = {

  setEnv: function(property, value) {
    this.env[property] = value
  },

  addCommand: function(name) {

    // throw error if name already taken
    var command = new Command('name')
    this.commands[name] = command
    return command
  },

  addValidator: function(name, logic) {
    this.validators[name] = logic
  },

  parse: function(input, stream) {

    input = this.cleanInput(input)
    var lexemes = input.split(' ')
    return this.parseLexemes(lexemes, stream)
  },

  cleanInput: function(input) {

    // server shell sends extra junk
    input = input.replace("\r\n", "\n")

    // remove trailing newline
    if (input.slice(-1) == "\n") {
      input = input.slice(0, input.length - 1)
    }

    // remove redundant spaces
    while (input.indexOf('  ') != -1) {
      input = input.replace('  ', ' ')
    }

    return input
  },

  parseLexemes: function(lexemes, stream) {

    var output = ''

    // cycle through commands looking for syntax match
    for (var index in this.commands) {

      var command = this.commands[index]

      // we clone lexemes because if the last syntax lexeme has a wildcard the
      // submitted lexeme corresponding to the last syntax lexeme ends up
      // getting subsequent submitted lexemes added to it
      if (command.try(this.validators, this.env, this.clone(lexemes), true)) {

        // command condition jazz goes here
        if (1) {

          result = command.try(this.validators, this.env, lexemes, false, stream)

          if (result) {
            return output + result
          }
        }
        else if(output) {

          return output
        }
      }
    }
  },

  urlToLexemes: function(url) {
    return url.split('/').slice(1)
  },

  clone: function(obj) {
    var newObj = (obj instanceof Array) ? [] : {};
    for (i in obj) {
      if (i == 'clone') continue;
      if (obj[i] && typeof obj[i] == "object") {
        newObj[i] = this.clone(obj[i]);
      } else newObj[i] = obj[i]
    } return newObj;
  }
}
{'fraction_non_alphanumeric': 0.13484617063059132, 'fraction_numerical': 0.0041457560549858176, 'mean_word_length': 2.353328456474031, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '451349', 'n_tokens_mistral': 1481, 'n_tokens_neox': 1398, 'n_words': 393}