<commit_before>"use strict"
const   mongoose = require('mongoose'),
    Concept = require('../../models/Concept');


module.exports = () => {


  let data = {}

  Concept.find()
    .sort('label')
    .select('label _id')
    .exec((err, collection)=>{
       if (err) throw err

        let dupes = {}
        collection.forEach((entry) => {
          if (!dupes[entry.label.replace(/ /g, "")]) {
            dupes[entry.label.replace(/ /g, "")] = true
          } else {
            //remove the entry
            // console.log('removing ',entry._id )
            Concept.findById(entry._id)
            .remove()
          }
        })

<commit_msg>edit function<commit_after>"use strict"
const   mongoose = require('mongoose'),
    Concept = require('../../models/Concept');


module.exports = () => {


  let data = {}

  Concept.find()
    .sort('label')
    .select('label _id')
    .exec((err, collection)=>{
       if (err) throw err

        let dupes = {}
        collection.forEach((entry) => {
          if (!dupes[entry.label.replace(/ /g, "")]) {
            dupes[entry.label.replace(/ /g, "")] = true
          } else {
            //remove the entry
            // console.log('removing ',entry._id )

          Concept.findById(entry._id, (err, concept) => {
              if (err) console.log(err)
              concept.remove()
            })
          }
        })


{'fraction_non_alphanumeric': 0.17782577393808496, 'fraction_numerical': 0.0, 'mean_word_length': 1.693798449612403, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6947763', 'n_tokens_mistral': 428, 'n_tokens_neox': 417, 'n_words': 80}