<commit_before>var fs = require('fs');
var path = require('path');
var os = require('os');

if(!global.__nodeProcessConcurrency){
  global.__nodeProcessConcurrency = {};
}

Shared = function(options){
  var options = options || {};

  var errors = {
    commitMessage: function(err){
      return "[npc:" + process.pid + "]: " +
             "ERROR COMMITTING PROCESS SHARED SPACE PAYLOAD: " + err;
    },
    retrieveMessage: function(err){
      return "[npc:" + process.pid + "]: " +
             "ERROR RETRIEVING PROCESS SHARED SPACE PAYLOAD: " + err;
    },
  }

  var priv = {
    swap: options['swap'] || path.join(__dirname, ('.__npc.swap')),
    //encoding: 'base64',
    encoding: 'ascii',

    payload: {},

    serialize: function(input){
      var output = {};

      for(var key in input){
        if(typeof input[key] == 'function'){
          output[key] = input[key].toString();
        } else {
          output[key] = input[key];
        }
      }

      return JSON.stringify(output, function(key, val){
        if(typeof val == 'function')    return val.toString();

        return val;
      });
    },

    deserialize: function(input){
      var parsed = JSON.parse(input);

      for(var id in parsed){
        if(parsed[id].match(/function\s*\(\)/)){
          // expand serialized functions in our data.
          eval_str = 'parsed[id] = ' + parsed[id];
          eval(eval_str);
        }
      }

      return parsed;
    },

    commit: function(){
      var payload = priv.serialize(priv.payload);
      fs.writeFile(priv.swap, payload, priv.encoding,
        function(err){
          if (err)  throw errors.commitMessage(err);
        }
      );<commit_msg>minor fix, don't process nonstrings<commit_after>var fs = require('fs');
var path = require('path');
var os = require('os');

if(!global.__nodeProcessConcurrency){
  global.__nodeProcessConcurrency = {};
}

Shared = function(options){
  var options = options || {};

  var errors = {
    commitMessage: function(err){
      return "[npc:" + process.pid + "]: " +
             "ERROR COMMITTING PROCESS SHARED SPACE PAYLOAD: " + err;
    },
    retrieveMessage: function(err){
      return "[npc:" + process.pid + "]: " +
             "ERROR RETRIEVING PROCESS SHARED SPACE PAYLOAD: " + err;
    },
  }

  var priv = {
    swap: options['swap'] || path.join(__dirname, ('.__npc.swap')),
    //encoding: 'base64',
    encoding: 'ascii',

    payload: {},

    serialize: function(input){
      var output = {};

      for(var key in input){
        if(typeof input[key] == 'function'){
          output[key] = input[key].toString();
        } else {
          output[key] = input[key];
        }
      }

      return JSON.stringify(output, function(key, val){
        if(typeof val == 'function')    return val.toString();

        return val;
      });
    },

    deserialize: function(input){
      var parsed = JSON.parse(input);

      for(var id in parsed){
        if(typeof parsed[id] == 'string' && parsed[id].match(/function\s*\(\)/)){
          // expand serialized functions in our data.
          eval_str = 'parsed[id] = ' + parsed[id];
          eval(eval_str);
        }
      }

      return parsed;
    },

    commit: function(){
      var payload = priv.serialize(priv.payload);
      fs.writeFile(priv.swap, payload, priv.encoding,
        function(err){
          if (err)  throw errors.commitMessage(err);
        }
      );
{'fraction_non_alphanumeric': 0.1571137620017457, 'fraction_numerical': 0.0011638056444573757, 'mean_word_length': 2.2742857142857145, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2585985', 'n_tokens_mistral': 1058, 'n_tokens_neox': 1030, 'n_words': 237}