<commit_before>var multiComment = false;
var multiLine = false;

var validOperators = {
    'is': ' === ',
    'not': ' !== ',
    'and':  ' && ',
    'or':  ' || ',
    'next':  '; ',
    'as':  ' = ',
    'more':  ' += ',
    'less':  ' -= ',
    'lots': ' *= ',
    'few': ' /= ',
    'very': ' var ',
    'smaller': ' < ',
    'bigger': ' > ',
    'smallerish': ' <= ',
    'biggerish': ' >= ',
    'notrly': ' ! ',
    'bigify': '++',
    'smallify': '--'
};

var assignOps = {
  'more': ' += ',
  'less': ' -= ',
  'lots': ' *= ',
  'few' : ' /= ',
  'is' : ' = '
}

var unaryOps = {
  'bigify': '++',
  'smallify': '--'
}

var controlFlowKeys = [
  'many',
  'much',
  'rly',
  'but',
  'notrly'
]

var valid = [
    'such',
    'wow',
    'wow&',
    'plz',
    '.plz',
    'dose',
    'very',
    'shh',
    'quiet',
    'loud',
    'rly',
    'notrly',
    'but',
    'many',
    'much',
    'so',
    'trained',
    'maybe',
    'pawse'
];

function containsUnary(keys)
{
  return unaryOps.hasOwnProperty(keys[0]) || unaryOps.hasOwnProperty(keys[1]);
}

var replacements = {};
replacements['dogeument']='document'
replacements['windoge']='window'
replacements['pawse']='debugger;'


module.exports = function parse (line) {

    // replace custom keywords
    Object.keys(replacements).forEach(function(key) {
        line = line.replace(new RegExp(key,'g'), replacements[key])
    });

    var keys = line.match(/'[^']+'|\S+/g);
    var statement = '';

    if (keys === null) return line + '\n';

    // not dogescript, such javascript
    if (valid.indexOf(keys[0]) === -1 && !assignOps.hasOwnProperty(keys[1]) && !containsUnary(keys) && keys[1] !== 'dose' || multiComment && keys[0] !== 'loud' || multiLine && keys[0] !== 'wow')
    {
      return line + '\n';
    }

    // trained use strict<commit_msg>refactor check for djs source (#130)<commit_after>var multiComment = false;
var multiLine = false;

var validOperators = {
    'is': ' === ',
    'not': ' !== ',
    'and':  ' && ',
    'or':  ' || ',
    'next':  '; ',
    'as':  ' = ',
    'more':  ' += ',
    'less':  ' -= ',
    'lots': ' *= ',
    'few': ' /= ',
    'very': ' var ',
    'smaller': ' < ',
    'bigger': ' > ',
    'smallerish': ' <= ',
    'biggerish': ' >= ',
    'notrly': ' ! ',
    'bigify': '++',
    'smallify': '--'
};

var assignOps = {
  'more': ' += ',
  'less': ' -= ',
  'lots': ' *= ',
  'few' : ' /= ',
  'is' : ' = '
}

var unaryOps = {
  'bigify': '++',
  'smallify': '--'
}

var controlFlowKeys = [
  'many',
  'much',
  'rly',
  'but',
  'notrly'
]

var valid = [
    'such',
    'wow',
    'wow&',
    'plz',
    '.plz',
    'dose',
    'very',
    'shh',
    'quiet',
    'loud',
    'rly',
    'notrly',
    'but',
    'many',
    'much',
    'so',
    'trained',
    'maybe',
    'pawse'
];

function containsUnary(keys)
{
  return unaryOps.hasOwnProperty(keys[0]) || unaryOps.hasOwnProperty(keys[1]);
}

/**
 * Determines if the parsed keys should be treated as dogescript source or not.
 */
function isDogescriptSource(keys)
{

  // starts the statement with a valid key
  if (valid.indexOf(keys[0]) !== -1)
  {
    return true;
  }

  // statement applying an assignment operator
  if ( assignOps.hasOwnProperty(keys[1]) )
  {
    return true;
  }

  // applying a unary operator
  if ( containsUnary(keys) )
  {
    return true;
  }

  // calling function on Object
  if ( keys[1] === 'dose' )
  {
    return true;
  }

  // ending a multi comment block
  if (multiComment && keys[0] === 'loud' )
  {
    return true;
  }

  // closing a multi-line object creation
  if ( multiLine && keys[0] === 'wow' )
  {
    return true;
  }

  // not valid dogescript
  return false;
}

var replacements = {};
replacements['dogeument']='document'
replacements['windoge']='window'
replacements['pawse']='debugger;'


module.exports = function parse (line) {

    // replace custom keywords
    Object.keys(replacements).forEach(function(key) {
        line = line.replace(new RegExp(key,'g'), replacements[key])
    });

    var keys = line.match(/'[^']+'|\S+/g);
    var statement = '';

    if (keys === null) return line + '\n';

    // not dogescript, such javascript
    if ( !isDogescriptSource(keys) )
    {
      return line + '\n';
    }

    // trained use strict
{'fraction_non_alphanumeric': 0.22593457943925233, 'fraction_numerical': 0.0044392523364485985, 'mean_word_length': 2.4330392943063353, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2040889', 'n_tokens_mistral': 1676, 'n_tokens_neox': 1633, 'n_words': 317}