<commit_before>var fs          = require('fs')
var rawData     = fs.read('emojis.json').toString()
var data        = JSON.parse(fs.read('emojis.json'))
var keys        = rawData.match(/"(.+)":/g).map(function(key){return key.replace(/"|:/g,'')})
var buildFailed = false
var passed      = function() { console.log("\x1B[92mPASSED\x1B[0m\n") }
var failed      = function() {
  console.log("\x1B[91mFAILED\x1B[0m\n")
  buildFailed = true
}

//
console.log("\nTEST: Correct number of emojis")

if(keys.length !== 884) {
  console.log("There are 884 emojis, but emojis.json has " + keys.length + " entries.")
  failed()
} else {
  passed()
}

//
console.log("TEST: No duplicated entries")

var arr = []
var dups = []

keys.forEach(function(key) {
  if(arr.indexOf(key) < 0) {
    arr.push(key)
  } else {
    dups.push(key)
  }
})

if(dups.length > 0) {
  dups.forEach(function(key) {
    console.log('There is more than one "' + key + '" in emojis.json.')
  })
  failed()
} else {
  passed()
}

//
console.log("TEST: No unnecessary keywords")

var unnecessities = []
var unnecessitiesInKeywords = []

keys.forEach(function(key) {
  data[key].forEach(function(keyword) {
    if(key.match(keyword)) {
      unnecessities.push([key, keyword])
    }

    var otherKeywords = data[key]
    otherKeywords.splice(data[key].indexOf(keyword), 1)
    otherKeywords.forEach(function(otherKeyword) {
      if(otherKeyword.match(keyword)) {
        unnecessitiesInKeywords.push([otherKeyword, keyword, key])
      }
    })
  })
})

if(unnecessities.length > 0 || unnecessitiesInKeywords.length > 0) {
  unnecessities.forEach(function(arr) {
    console.log('"' + arr[1] + '" is unnecessary as it is already part of "' + arr[0] + '" and will be matched.')
  })
  unnecessitiesInKeywords.forEach(function(arr) {
    console.log('"' + arr[1] + '" is unnecessary as a "' + arr[2] + '" already has a keyword "' + arr[0] + '".')
  })

  failed()
} else {
  passed()
}

//
console.log("TEST: Line format")

var offenses = []<commit_msg>Fix test, keywords are being removed cause references<commit_after>var fs          = require('fs')
var rawData     = fs.read('emojis.json').toString()
var data        = JSON.parse(fs.read('emojis.json'))
var keys        = rawData.match(/"(.+)":/g).map(function(key){return key.replace(/"|:/g,'')})
var buildFailed = false
var passed      = function() { console.log("\x1B[92mPASSED\x1B[0m\n") }
var failed      = function() {
  console.log("\x1B[91mFAILED\x1B[0m\n")
  buildFailed = true
}

//
console.log("\nTEST: Correct number of emojis")

if(keys.length !== 884) {
  console.log("There are 884 emojis, but emojis.json has " + keys.length + " entries.")
  failed()
} else {
  passed()
}

//
console.log("TEST: No duplicated entries")

var arr = []
var dups = []

keys.forEach(function(key) {
  if(arr.indexOf(key) < 0) {
    arr.push(key)
  } else {
    dups.push(key)
  }
})

if(dups.length > 0) {
  dups.forEach(function(key) {
    console.log('There is more than one "' + key + '" in emojis.json.')
  })
  failed()
} else {
  passed()
}

//
console.log("TEST: No unnecessary keywords")

var unnecessities = []
var unnecessitiesInKeywords = []

keys.forEach(function(key) {
  data[key].forEach(function(keyword) {
    if(key.match(keyword)) {
      unnecessities.push([key, keyword])
    }

    var otherKeywords = data[key].slice()
    otherKeywords.splice(data[key].indexOf(keyword), 1)
    otherKeywords.forEach(function(otherKeyword) {
      if(otherKeyword.match(keyword)) {
        unnecessitiesInKeywords.push([otherKeyword, keyword, key])
      }
    })
  })
})

if(unnecessities.length > 0 || unnecessitiesInKeywords.length > 0) {
  unnecessities.forEach(function(arr) {
    console.log('"' + arr[1] + '" is unnecessary as it is already part of "' + arr[0] + '" and will be matched.')
  })
  unnecessitiesInKeywords.forEach(function(arr) {
    console.log('"' + arr[1] + '" is unnecessary as a "' + arr[2] + '" already has a keyword "' + arr[0] + '".')
  })

  failed()
} else {
  passed()
}

//
console.log("TEST: Line format")

var offenses = []
{'fraction_non_alphanumeric': 0.18025540275049115, 'fraction_numerical': 0.012770137524557957, 'mean_word_length': 3.422366992399566, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7066800', 'n_tokens_mistral': 1487, 'n_tokens_neox': 1450, 'n_words': 281}