<commit_before>var tokens = require('./tokens').tokens;

//Some helper functions
//These should go somewhere else

var isBlank = function (token) {
  if (token.match(/\s/) || token.match(/\n/) || token.match(/\t/)){
    return true;
  } else {
    return false;
  }
};

var isPrefix = function (needle, haystack) {
  if (haystack.indexOf(needle) === 0){
    return true;
  } else {
    return false;
  }
};

var inSet = function (needle, haystack) {
  if (haystack.indexOf(needle) >= 0){
    return true;
  } else {
    return false;
  }
};

var states = {
  lexStatement: function (lexer) {
    while (true) {
      //TODO: clean this up. Don't need restOfString.
      var nextChar = lexer.inputArr.charAt(lexer.pos);
      var restOfString = lexer.inputArr.substring(lexer.pos);
      if (isPrefix(tokens.openBrace, restOfString)) {
        lexer.pos = lexer.start;
        return states.lexSelector;
      } else if (isPrefix(tokens.atStart, restOfString)) {
        return states.lexAt;
      } else if (isPrefix(tokens.semicolon, restOfString)) {
        lexer.emit('declaration');
        return states.lexSemicolon;
      } else if (isPrefix(tokens.closeBrace, restOfString)) {
        lexer.emit('declaration');
        return states.lexCloseBrace;
      } else if (isBlank(nextChar) && lexer.start === lexer.pos){<commit_msg>removed string duplication for major performance gain<commit_after>var tokens = require('./tokens').tokens;

//Some helper functions
//These should go somewhere else

var isBlank = function (token) {
  if (token.match(/\s/) || token.match(/\n/) || token.match(/\t/)){
    return true;
  } else {
    return false;
  }
};

var isPrefix = function (needle, haystack) {
  if (haystack.indexOf(needle) === 0){
    return true;
  } else {
    return false;
  }
};

var inSet = function (needle, haystack) {
  if (haystack.indexOf(needle) >= 0){
    return true;
  } else {
    return false;
  }
};

var states = {
  lexStatement: function (lexer) {
    while (true) {
      //TODO: clean this up. Don't need restOfString.
      var nextChar = lexer.inputArr.charAt(lexer.pos);
      if (nextChar === tokens.openBrace) {
        lexer.pos = lexer.start;
        return states.lexSelector;
      } else if (nextChar === tokens.atStart) {
        return states.lexAt;
      } else if (nextChar === tokens.semicolon) {
        lexer.emit('declaration');
        return states.lexSemicolon;
      } else if (nextChar === tokens.closeBrace) {
        lexer.emit('declaration');
        return states.lexCloseBrace;
      } else if (isBlank(nextChar) && lexer.start === lexer.pos){
{'fraction_non_alphanumeric': 0.14324116743471582, 'fraction_numerical': 0.0015360983102918587, 'mean_word_length': 2.8084795321637426, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2577512', 'n_tokens_mistral': 879, 'n_tokens_neox': 853, 'n_words': 206}