<commit_before>'use strict'

const eachOf = require('async').eachOf
const elasticlunr = require('elasticlunr')
const debug = require('debug')('metalsmith-elasticlunr')

const plugin = params => {
  const mergeOptions = user_opts => {
    const defaults = {
      indexingKey: 'index',
      ref: 'path',
      fields: [ 'title', 'contents' ],
      destFile: 'index.json',
      bootstrap: null,
      preprocess: null
    }

    return Object.assign(defaults, user_opts)
  }

  const initializeIndex = options => {
    // this refers to the index object
    return elasticlunr(function () {
      this.setRef(options.ref)

      options.fields.forEach(field => {
        this.addField(field)
      })

      if (options.bootstrap != null) {
        options.bootstrap.call(this)
      }
    })
  }

  const indexFile = (file, path, done) => {
    if (file[options.indexingKey]) {
      const doc = {}

      if (options.ref === 'path') {
        doc[options.ref] = path
      } else {
        doc[options.ref] = file[options.ref]
      }

      options.fields.forEach(field => {
        let val = file[field].toString()

        if (field === 'contents' && typeof options.preprocess === 'function') {
          val = options.preprocess.call(this, file[field].toString())
        }

        doc[field] = val
      })

      index.addDoc(doc)

      debug(`indexing ${path}`)
      done()
    } else {
      debug(`skipped ${path}`)
      done()
    }
  }

  const options = mergeOptions(params)
  const index = initializeIndex(options)

  return (files, metalsmith, done) => {
    eachOf(files, indexFile, err => {
      if (err) console.error(err)
      files[options.destFile] = {
        contents: new Buffer(JSON.stringify(index.toJSON()))
      }
      debug('elasticlunr indexing completed')
      done()<commit_msg>reducing code complexity<commit_after>'use strict'

const eachOf = require('async').eachOf
const elasticlunr = require('elasticlunr')
const debug = require('debug')('metalsmith-elasticlunr')

const defaults = {
  indexingKey: 'index',
  ref: 'path',
  fields: [ 'title', 'contents' ],
  destFile: 'index.json',
  bootstrap: null,
  preprocess: null
}

const mergeOptions = user_opts => {
  return Object.assign({}, defaults, user_opts)
}

const initializeIndex = options => {
  // this refers to the index object
  return elasticlunr(function () {
    this.setRef(options.ref)

    options.fields.forEach(field => {
      this.addField(field)
    })

    if (options.bootstrap != null) {
      options.bootstrap.call(this)
    }
  })
}

const indexerFn = (options, index) => {
  return (file, path, done) => {
    if (file[options.indexingKey]) {
      const doc = {}

      if (options.ref === 'path') {
        doc[options.ref] = path
      } else {
        doc[options.ref] = file[options.ref]
      }

      options.fields.forEach(field => {
        let val = file[field].toString()

        if (field === 'contents' && typeof options.preprocess === 'function') {
          val = options.preprocess.call(this, file[field].toString())
        }

        doc[field] = val
      })

      index.addDoc(doc)

      debug(`indexing ${path}`)
      done()
    } else {
      debug(`skipped ${path}`)
      done()
    }
  }
}

const plugin = params => {
  const options = mergeOptions(params)
  const index = initializeIndex(options)

  return (files, metalsmith, done) => {
    const indexFile = indexerFn(options, index)

    eachOf(files, indexFile, err => {
      if (err) console.error(err)
      files[options.destFile] = {
        contents: new Buffer(JSON.stringify(index.toJSON()))
      }
      debug('elasticlunr indexing completed')
      done()
{'fraction_non_alphanumeric': 0.1458503401360544, 'fraction_numerical': 0.0, 'mean_word_length': 2.611001964636542, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5849297', 'n_tokens_mistral': 1166, 'n_tokens_neox': 1122, 'n_words': 243}