<commit_before>var fs          = require('fs')
var rawData     = fs.read('emojis.json').toString()
var buildFailed = false
var escapeStr   = require('escape-string-regexp')
var passed      = function() { console.log("\x1B[92mPASSED\x1B[0m\n") }
var failed      = function() {
  console.log("\x1B[91mFAILED\x1B[0m\n")
  buildFailed = true
}
var reveal = function() {
  if(buildFailed) {
    console.log(":cry: \x1B[91mNo good, something failed.\x1B[0m :boom:\n")
    phantom.exit(buildFailed)
  } else {
    console.log(":sparkles: \x1B[96mWho's awesome? You're awesome!\x1B[0m :+1:\n")
    phantom.exit()
  }
}

//
console.log("\nTEST: Correct JSON format")

try {
  var data = JSON.parse(fs.read('emojis.json'))
  var keys = Object.keys(data)
  keys.splice(keys.indexOf('keys'), 1)
  passed()
} catch(e) {
  console.log('Invalid JSON format. See the CONTRIBUTING doc for reference.')
  failed()
  reveal()
}

//
console.log("TEST: Correct number of emojis")

var emojiNumber = 910 // update this number when emoji are added

if(keys.length !== emojiNumber) {
  console.log("There are " + emojiNumber + " emojis, but emojis.json has " + keys.length + " entries.")
  failed()
} else {
  passed()
}

//
console.log("TEST: Ordered keys are up to date")

if(data.keys.length !== emojiNumber) {
  console.log("There are " + emojiNumber + " emojis, but keys contains " + data.keys.length + " emojis.")
  failed()
} else {
  passed()
}

//
console.log("TEST: No duplicated entries")

var arr = []
var dups = []

keys.forEach(function(key) {
  if(arr.indexOf(key) < 0) {
    arr.push(key)
  } else {
    dups.push(key)
  }
})

if(dups.length > 0) {
  dups.forEach(function(key) {
    console.log('There is more than one "' + key + '" in emojis.json.')
  })
  failed()
} else {
  passed()
}

//
console.log("TEST: No unnecessary keywords")

var unnecessities = []
var unnecessitiesInKeywords = []

keys.forEach(function(key) {
  data[key]["keywords"].forEach(function(keyword) {
    keyword = escapeStr(keyword)<commit_msg>Fix dupes not really being tested (thought I fixed this ages ago?<commit_after>var fs          = require('fs')
var rawData     = fs.read('emojis.json').toString()
var buildFailed = false
var escapeStr   = require('escape-string-regexp')
var passed      = function() { console.log("\x1B[92mPASSED\x1B[0m\n") }
var failed      = function() {
  console.log("\x1B[91mFAILED\x1B[0m\n")
  buildFailed = true
}
var reveal = function() {
  if(buildFailed) {
    console.log(":cry: \x1B[91mNo good, something failed.\x1B[0m :boom:\n")
    phantom.exit(buildFailed)
  } else {
    console.log(":sparkles: \x1B[96mWho's awesome? You're awesome!\x1B[0m :+1:\n")
    phantom.exit()
  }
}

//
console.log("\nTEST: Correct JSON format")

try {
  var data = JSON.parse(fs.read('emojis.json'))
  var keys = Object.keys(data)
  keys.splice(keys.indexOf('keys'), 1)
  passed()
} catch(e) {
  console.log('Invalid JSON format. See the CONTRIBUTING doc for reference.')
  failed()
  reveal()
}

//
console.log("TEST: Correct number of emojis")

var emojiNumber = 916 // update this number when emoji are added

if(keys.length !== emojiNumber) {
  console.log("There are " + emojiNumber + " emojis, but emojis.json has " + keys.length + " entries.")
  failed()
} else {
  passed()
}

//
console.log("TEST: Ordered keys are up to date")

if(data.keys.length !== emojiNumber) {
  console.log("There are " + emojiNumber + " emojis, but keys contains " + data.keys.length + " emojis.")
  failed()
} else {
  passed()
}

//
console.log("TEST: No duplicated entries")

var arr = []
var dups = []
var keysFromRawData = rawData.match(/\".+\": {/g)

keysFromRawData.forEach(function(key) {
  if(arr.indexOf(key) < 0) {
    arr.push(key)
  } else {
    dups.push(key.replace(/\"|\:|\s|\{/g, ""))
  }
})

if(dups.length > 0) {
  dups.forEach(function(key) {
    console.log('There is more than one "' + key + '" in emojis.json.')
  })
  failed()
} else {
  passed()
}

//
console.log("TEST: No unnecessary keywords")

var unnecessities = []
var unnecessitiesInKeywords = []

keys.forEach(function(key) {
  data[key]["keywords"].forEach(function(keyword) {
    keyword = escapeStr(keyword)
{'fraction_non_alphanumeric': 0.1731830175101943, 'fraction_numerical': 0.012952746461981291, 'mean_word_length': 3.597574421168688, 'pattern_counts': {'":': 5, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3444353', 'n_tokens_mistral': 1559, 'n_tokens_neox': 1527, 'n_words': 328}