<commit_before>var config = {
  github: {
    username: 'mcwhittemore'
  }
}

var GISTDB = require('../gist-db')

var fileInit = function (file) {

  // init custom group object
  file.groups = {}

  // define groups and their regex include rules
  var group_rules = {
    'blog': /^Blog_/,
    'project': /^Project_/,
    'icon': /^Icon_$/,
    'about': /^BlogAboutPage.md$/
  }

  // get group names
  var groups = Object.keys(group_rules)

  // set file as excluded as we only want to include it if it has a group
  var include = false

  for (var i = 0; i < groups.length; i++) {
    var group = groups[i]
    var rule = group_rules[group]

    // check if filename matches regex rule
    if (file.filename.search(rule) > -1) {
      file.groups[group] = true // set included in group as true
      include = true // set include file as true
    }else {
      file.groups[group] = false
    }
  }

  if (include) {
    return file
  }else {
    return undefined
  }
}

var fileSave = function (file, callback) {
  file.html = file.raw
  callback(file)
}

var _db = GISTDB(config, fileInit, fileSave)

_db.event.on('github_error', function (err, res) {
  console.log('github error')
  console.log(err)
  console.log()
})

_db.event.on('file_error', function (err, file) {
  console.log('file error')
  console.log('FILE: ' + file.id)
  console.log('ERROR: ' + err.code)
  console.log()
})

_db.event.on('refreshing', function () {
  // MIGHT WANT TO LOCK DOWN THINGS FOR A BIT
  console.log('LETS DO THIS')
})

_db.event.on('refreshed', function (err) {
  console.log('refresh done')
  _db().each(function (file) {
    console.log(file.id)
    if (!file.error) {
      console.log('RAW: ' + file.raw.length + ' | HTML: ' + file.html.length)
      console.log()
    }else {
      console.log('ERROR')
      console.log()
    }
  })
})<commit_msg>Fix some linting errors, "handle" error<commit_after>var config = {
  github: {
    username: 'mcwhittemore'
  }
}

var GISTDB = require('../gist-db')

var fileInit = function (file) {
  // init custom group object
  file.groups = {}

  // define groups and their regex include rules
  var groupRules = {
    'blog': /^Blog_/,
    'project': /^Project_/,
    'icon': /^Icon_$/,
    'about': /^BlogAboutPage.md$/
  }

  // get group names
  var groups = Object.keys(groupRules)

  // set file as excluded as we only want to include it if it has a group
  var include = false

  for (var i = 0; i < groups.length; i++) {
    var group = groups[i]
    var rule = groupRules[group]

    // check if filename matches regex rule
    if (file.filename.search(rule) > -1) {
      file.groups[group] = true // set included in group as true
      include = true // set include file as true
    } else {
      file.groups[group] = false
    }
  }

  if (include) {
    return file
  } else {
    return undefined
  }
}

var fileSave = function (file, callback) {
  file.html = file.raw
  callback(file)
}

var _db = GISTDB(config, fileInit, fileSave)

_db.event.on('github_error', function (err, res) {
  console.log('github error')
  console.log(err)
  console.log()
})

_db.event.on('file_error', function (err, file) {
  console.log('file error')
  console.log('FILE: ' + file.id)
  console.log('ERROR: ' + err.code)
  console.log()
})

_db.event.on('refreshing', function () {
  // MIGHT WANT TO LOCK DOWN THINGS FOR A BIT
  console.log('LETS DO THIS')
})

_db.event.on('refreshed', function (err) {
  if (err) {
    console.log(err)
  }
  console.log('refresh done')
  _db().each(function (file) {
    console.log(file.id)
    if (!file.error) {
      console.log('RAW: ' + file.raw.length + ' | HTML: ' + file.html.length)
      console.log()
    } else {
      console.log('ERROR')
      console.log()
    }
  })
})
{'fraction_non_alphanumeric': 0.15376429901569566, 'fraction_numerical': 0.0010641127959563714, 'mean_word_length': 2.722772277227723, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6760051', 'n_tokens_mistral': 1337, 'n_tokens_neox': 1307, 'n_words': 348}