<commit_before>var util = require('util')

// Matches "blah = function("
var varFunc   = /([\w.]+)\s*=\s*function\([^)]*\)[\w\n\r]*\{/g

// Matches "function blah("
var namedFunc = /(?:^|[\W])function\s+(\w+)\([^)]*\)[\w\n\r]*\{/g

// Applies one of the function matching regexes to the source and returns a
// new version of the source (updates `functions` with the signatures of
// found functions)
var applyRegex = function(regex, source, filename, functions){
  // Reset the regex
  regex.lastIndex = 0
  var match
  while((match = regex.exec(source)) !== null){
    var definition = match[0],// Entire match is the function definition
        name       = match[1],// Just the function's name
        sig        = util.inspect(filename+':'+name),// Build the signature
        index      = match.index
    // Append the `_rdce_called` call to the definition
    var newDefinition = definition+'_rdce_called('+sig+');'
    // Add the function to the registry
    functions[sig] = true
    // Then replace it
    source = source.substr(0, index) + newDefinition + source.substr(index + definition.length)<commit_msg>Update comments<commit_after>var util = require('util')

// Matches "blah = function(...) {"
var varFunc   = /([\w.]+)\s*=\s*function\([^)]*\)[\w\n\r]*\{/g

// Matches "function blah(...) {"
var namedFunc = /(?:^|[\W])function\s+(\w+)\([^)]*\)[\w\n\r]*\{/g

// Applies one of the function matching regexes to the source and returns a
// new version of the source (updates `functions` with the signatures of
// found functions)
var applyRegex = function(regex, source, filename, functions){
  // Reset the regex
  regex.lastIndex = 0
  var match
  while((match = regex.exec(source)) !== null){
    var definition = match[0],// Entire match is the function definition
        name       = match[1],// Just the function's name
        sig        = util.inspect(filename+':'+name),// Build the signature
        index      = match.index
    // Append the `_rdce_called` call to the definition
    var newDefinition = definition+'_rdce_called('+sig+');'
    // Add the function to the registry
    functions[sig] = true
    // Then replace it
    source = source.substr(0, index) + newDefinition + source.substr(index + definition.length)
{'fraction_non_alphanumeric': 0.16429840142095914, 'fraction_numerical': 0.003552397868561279, 'mean_word_length': 3.1113138686131387, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1241724', 'n_tokens_mistral': 688, 'n_tokens_neox': 678, 'n_words': 212}