<commit_before>'use strict'
var merge = require('merge')
var yaml = require('js-yaml')
var logger = require('./logger')
var fs = require('fs')
var cfgFiles = null

function mergeConfigs (cfgs) {
  var conf = cfgs[0]
  for (var config in cfgs) {
    var tmp = merge.recursive(false, conf, cfgs[config])
    if (!tmp.patterns) {
      logger.error('missing patterns section in config ' + tmp._fileName)
    } else {
      tmp.patterns = cfgs[config].patterns.concat(conf.patterns)  
    }
    conf = tmp
  }
  return conf
}

function notifyFileChange (cur, prev) {
  if (cur.mtime > prev.mtime) {
    this.cb(this.file)
  }
}

function unwatchConfigs (files, notifyCallback) {
  files.forEach(function (f) {
    fs.unwatchFile(f)
  })
}
function watchConfigs (files, notifyCallback) {
  files.forEach(function (f) {
    fs.watchFile(f, notifyFileChange.bind({cb: notifyCallback, file: f}))
  })
}
function loadConfigFiles (files, notifyCallback) {
  if (cfgFiles !== null) {
    unwatchConfigs(cfgFiles)
  }
  var configs = files.map(function (file) {
    logger.debug('merge pattern file ' + file)
    var cfg = {}
    try {
      cfg = yaml.load(fs.readFileSync(file, 'utf8'))
      cfg._fileName=file
    } catch (e) {
      logger.error('ignoring pattern file ' + file + ' ' + e)
      if (e.reason && e.mark) {
        logger.error('Error parsing file: ' + file + ' ' + e.reason + ' line:' + e.mark.line + ' column:' + e.mark.columns)
      } else {
        // console.log(error.stack)
        logger.error('Error parsing file: ' + file + ' ' + e + ' ' + e.stack)
      }
    } finally {
      // console.log(config)
      return cfg
    }
  })
  if (notifyCallback) {
    try {
      cfgFiles = files
      watchConfigs(cfgFiles, notifyCallback)
    } catch (err) {
      logger.error('error watching pattern file:' + err)
    }
  }
  return mergeConfigs(configs)
}

// console.log(loadConfigFiles(['./patterns.yml']))
// module.exports.mergeConfigs = mergeConfigs
module.exports = loadConfigFiles<commit_msg>move from fs.watchFile() to fs.watch() because of inotify on Linux<commit_after>'use strict'
var merge = require('merge')
var yaml = require('js-yaml')
var logger = require('./logger')
var fs = require('fs')
var cfgFiles = null

function mergeConfigs (cfgs) {
  var conf = cfgs[0]
  if (!conf.patterns) {
    logger.error('missing patterns section in config ' + conf._fileName)
    conf.patterns = []
  }
  for (var config in cfgs) {
    var tmp = merge.recursive(false, conf, cfgs[config])
    if (!tmp.patterns) {
      logger.error('missing patterns section in config ' + tmp._fileName)
      tmp.patterns = []
    }
    tmp.patterns = cfgs[config].patterns.concat(conf.patterns)  
    conf = tmp
  }
  return conf
}

function notifyFileChange (event, file) {
  if (event === 'change') {
    this.cb(file)
  }
}

function watchConfigs (files, notifyCallback) {
  files.forEach(function (f) {
    console.log('start watching ' + f )
    fs.watch(f, {persistent: false, recursive: false}, notifyFileChange.bind({cb: notifyCallback, file: f}))
  })
}
function loadConfigFiles (files, notifyCallback) {
  var configs = files.map(function (file) {
    logger.debug('merge pattern file ' + file)
    var cfg = {}
    try {
      cfg = yaml.load(fs.readFileSync(file, 'utf8'))
      cfg._fileName=file
    } catch (e) {
      logger.error('ignoring pattern file ' + file + ' ' + e)
      if (e.reason && e.mark) {
        logger.error('Error parsing file: ' + file + ' ' + e.reason + ' line:' + e.mark.line + ' column:' + e.mark.columns)
      } else {
        // console.log(error.stack)
        logger.error('Error parsing file: ' + file + ' ' + e + ' ' + e.stack)
      }
    } finally {
      // console.log(config)
      return cfg
    }
  })
  if (notifyCallback && cfgFiles === null) {
    try {
      cfgFiles = files

      watchConfigs(cfgFiles, notifyCallback)
    } catch (err) {
      logger.error('error watching pattern file:' + err)
    }
  }
  return mergeConfigs(configs)
}

// console.log(loadConfigFiles(['./patterns.yml']))
// module.exports.mergeConfigs = mergeConfigs
module.exports = loadConfigFiles
{'fraction_non_alphanumeric': 0.1478702807357212, 'fraction_numerical': 0.000968054211035818, 'mean_word_length': 2.917535545023697, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1426902', 'n_tokens_mistral': 1363, 'n_tokens_neox': 1353, 'n_words': 313}