<commit_before>fs = require 'fs'
sysPath = require 'path'
RegExp.quote = require 'regexp-quote'


class KeywordProcessor
  @instance: null
  brunchPlugin: yes
  globalRE: null
  globalMap: null
  packageInfoFilePath: "package.json"
  lastCompileDate: null
  lastCompileResult: null


  constructor: (@config = {}) ->
    KeywordProcessor.instance = @
    return unless @config.keyword
    path = sysPath.resolve(@config.paths?.public ? 'public')
    if (pip = @config.keyword?.jsonPackageFilePath)
      @packageInfoFilePath = pip
    if @packageInfoFilePath
      @packageInfoFilePath = fs.realpathSync(@packageInfoFilePath)
    @publicPath = path
    @keywordConfig = @config.keyword
    @filePattern = @keywordConfig.filePattern ? /\.(js|css|html)$/
    @keywordMap = @keywordConfig.map ? {}


  generateDefaultMap: (compilationDate = new Date()) ->
    map = {}
    packageInfo = {}
    if fs.existsSync(@packageInfoFilePath)
      packageInfo = JSON.parse fs.readFileSync(@packageInfoFilePath).toString()
      for keyword in ["version", "name"]
        if packageInfo[keyword]
          map[keyword] = packageInfo[keyword]
        else
          console.error "#{fs.basename @packageInfoFilePath} need a #{keyword}"
    map['date'] = @getDateValue compilationDate
    map['timestamp'] = @getTimestampValue compilationDate
    map


  getDateValue: (compilationDate = @lastCompileDate) ->
    compilationDate.toUTCString()


  getTimestampValue: (compilationDate = @lastCompileDate) ->
    "#{compilationDate.getTime()}"



  readdirRecursive: (folder, callback) ->
    todo = 0
    doneCalled = no
    fileList = []
    done = (err) ->
      return if doneCalled
      doneCalled = yes
      if callback
        callback err, if err then [] else fileList
      else if err
        throw err
    check = (decr = no) ->
      todo-- if decr
      done() if todo is 0
    readOneDir = (folder) ->
      todo++
      fs.readdir folder, (err, files) ->
        if err
          done err
        else
          todo += files.length
          for file in files
            ((fullPath) ->
              fs.stat fullPath, (err, stat) ->
                if err
                  done err
                else
                  if stat.isDirectory()
                    todo--
                    readOneDir fullPath
                  else
                    fileList.push fullPath
                    check yes
            )(sysPath.join(folder, file))
          check yes
    readOneDir folder


  readdirRecursiveSync: (path, filterFunc = -> yes) ->
    unless fs.existsSync(path)
      return []
    unless fs.statSync(path).isDirectory()
      if filterFunc(fs.basename(path), fs.dirname(path))
        [path]
      else
        []
    else
      res = []<commit_msg>Fix path handling exceptions resulting from wrong module use<commit_after>fs = require 'fs'
sysPath = require 'path'
RegExp.quote = require 'regexp-quote'


class KeywordProcessor
  @instance: null
  brunchPlugin: yes
  globalRE: null
  globalMap: null
  packageInfoFilePath: "package.json"
  lastCompileDate: null
  lastCompileResult: null


  constructor: (@config = {}) ->
    KeywordProcessor.instance = @
    return unless @config.keyword
    path = sysPath.resolve(@config.paths?.public ? 'public')
    if (pip = @config.keyword?.jsonPackageFilePath)
      @packageInfoFilePath = pip
    if @packageInfoFilePath
      @packageInfoFilePath = fs.realpathSync(@packageInfoFilePath)
    @publicPath = path
    @keywordConfig = @config.keyword
    @filePattern = @keywordConfig.filePattern ? /\.(js|css|html)$/
    @keywordMap = @keywordConfig.map ? {}


  generateDefaultMap: (compilationDate = new Date()) ->
    map = {}
    packageInfo = {}
    if fs.existsSync(@packageInfoFilePath)
      packageInfo = JSON.parse fs.readFileSync(@packageInfoFilePath).toString()
      for keyword in ["version", "name"]
        if packageInfo[keyword]
          map[keyword] = packageInfo[keyword]
        else
          console.error "#{sysPath.basename @packageInfoFilePath} need a #{keyword}"
    map['date'] = @getDateValue compilationDate
    map['timestamp'] = @getTimestampValue compilationDate
    map


  getDateValue: (compilationDate = @lastCompileDate) ->
    compilationDate.toUTCString()


  getTimestampValue: (compilationDate = @lastCompileDate) ->
    "#{compilationDate.getTime()}"



  readdirRecursive: (folder, callback) ->
    todo = 0
    doneCalled = no
    fileList = []
    done = (err) ->
      return if doneCalled
      doneCalled = yes
      if callback
        callback err, if err then [] else fileList
      else if err
        throw err
    check = (decr = no) ->
      todo-- if decr
      done() if todo is 0
    readOneDir = (folder) ->
      todo++
      fs.readdir folder, (err, files) ->
        if err
          done err
        else
          todo += files.length
          for file in files
            ((fullPath) ->
              fs.stat fullPath, (err, stat) ->
                if err
                  done err
                else
                  if stat.isDirectory()
                    todo--
                    readOneDir fullPath
                  else
                    fileList.push fullPath
                    check yes
            )(sysPath.join(folder, file))
          check yes
    readOneDir folder


  readdirRecursiveSync: (path, filterFunc = -> yes) ->
    unless fs.existsSync(path)
      return []
    unless fs.statSync(path).isDirectory()
      if filterFunc(sysPath.basename(path), sysPath.dirname(path))
        [path]
      else
        []
    else
      res = []
{'fraction_non_alphanumeric': 0.1036084315827081, 'fraction_numerical': 0.0007145409074669524, 'mean_word_length': 2.1686474250141483, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2922470', 'n_tokens_mistral': 1594, 'n_tokens_neox': 1547, 'n_words': 396}