<commit_before>// -*- indent-tabs-mode: nil; js2-basic-offset: 2 -*-

var fs = require('fs');
var path = require('path');
var mkdirp = require('mkdirp');
var rmRSync = require('../rm').rmRSync;
var crypto = require('crypto');

function FileStorage(args) {
  this.initialize(args);
}
FileStorage.prototype = {
  initialize: function(args) {
    this.name = args.name;
    this.basePath = args.basePath || '/tmp';
  },

  get directoryPath() {
    if (!this._directoryPath) {
      this._directoryPath = paht.join(this.basePath, this.name);
    }
    return this._directoryPath;
  },

  // because document id can be too long, we should use a short hash
  // string instead of full id.
  getFileName: function(id) {
    var shasum = crypto.createHash('sha1');
    shasum.update(id);
    return shasum.digest('hex');
  },

  getFilePath: function(id) {
    var filename = this.getFileName(id);
    return path.join(this.directoryPath, filename);
  },

  saveSync: function(document) {
    if (!this.directoryExistsSync())
      mkdirp.sync(this.databaseDirectory);

    var filePath = this.getFilePath(document.id);
    if (path.existsSync(filePath))
      fs.unlinkSync(filePath);

    var contents = JSON.stringify(document);
    var file = fs.openSync(filePath, 'w', 0644);
    fs.writeSync(file, contents);
    fs.closeSync(file);
  },

  readSync: function(id) {
    if (!this.directoryExistsSync())
      return null;

    var filePath = this.getFilePath(document.id);
    if (path.existsSync(filePath))
      return null;

    var contents = fs.readFileSync(filePath, 'UTF-8');
    return JSON.parse(contents);
  },

  deleteSync: function(id) {
    if (!this.directoryExistsSync())
      return;

    var filePath = this.getFilePath(document.id);
    if (path.existsSync(filePath))
      fs.unlinkSync(filePath);
  },

  clearSync: function() {
    if (this.directoryExistsSync())
      rmRSync(this.directoryPath);
  },

  directoryExistsSync: function() {
    return path.existsSync(this.directoryPath);
  }
};
exports.FileStorage = FileStorage;


function MemoryStorage() {
  this._storage = {};
}
MemoryStorage.prototype = {<commit_msg>Rename methods: getXXX => idToXXX<commit_after>// -*- indent-tabs-mode: nil; js2-basic-offset: 2 -*-

var fs = require('fs');
var path = require('path');
var mkdirp = require('mkdirp');
var rmRSync = require('../rm').rmRSync;
var crypto = require('crypto');

function FileStorage(args) {
  this.initialize(args);
}
FileStorage.prototype = {
  initialize: function(args) {
    this.name = args.name;
    this.basePath = args.basePath || '/tmp';
  },

  get directoryPath() {
    if (!this._directoryPath) {
      this._directoryPath = paht.join(this.basePath, this.name);
    }
    return this._directoryPath;
  },

  // because document id can be too long, we should use a short hash
  // string instead of full id.
  idToFileName: function(id) {
    var shasum = crypto.createHash('sha1');
    shasum.update(id);
    return shasum.digest('hex');
  },

  idToFilePath: function(id) {
    var filename = this.idToFileName(id);
    return path.join(this.directoryPath, filename);
  },

  saveSync: function(document) {
    if (!this.directoryExistsSync())
      mkdirp.sync(this.databaseDirectory);

    var filePath = this.idToFilePath(document.id);
    if (path.existsSync(filePath))
      fs.unlinkSync(filePath);

    var contents = JSON.stringify(document);
    var file = fs.openSync(filePath, 'w', 0644);
    fs.writeSync(file, contents);
    fs.closeSync(file);
  },

  readSync: function(id) {
    if (!this.directoryExistsSync())
      return null;

    var filePath = this.idToFilePath(document.id);
    if (path.existsSync(filePath))
      return null;

    var contents = fs.readFileSync(filePath, 'UTF-8');
    return JSON.parse(contents);
  },

  deleteSync: function(id) {
    if (!this.directoryExistsSync())
      return;

    var filePath = this.idToFilePath(document.id);
    if (path.existsSync(filePath))
      fs.unlinkSync(filePath);
  },

  clearSync: function() {
    if (this.directoryExistsSync())
      rmRSync(this.directoryPath);
  },

  directoryExistsSync: function() {
    return path.existsSync(this.directoryPath);
  }
};
exports.FileStorage = FileStorage;


function MemoryStorage() {
  this._storage = {};
}
MemoryStorage.prototype = {
{'fraction_non_alphanumeric': 0.14573679332715478, 'fraction_numerical': 0.0037071362372567192, 'mean_word_length': 3.4921956295525494, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2096300', 'n_tokens_mistral': 1408, 'n_tokens_neox': 1363, 'n_words': 285}