<commit_before>/* File system primitives.
 * Copyright © 2011 <NAME>, <NAME>. All rights reserved.
 * The following code is covered by the GPLv2 license. */

var nodefs = require ('fs'),
    nodepath = require ('path'),
    util = require('util'),
    CodeMirrorServer = require('operational-transformation').CodeMirrorServer,
    async = require('async');

var driver = require('./driver'),  // Driver.
    type = require('./type');      // Type system.



// Gather information for the profiler.
//
// Numbers are to computer science what a telescope is to an astrophysicist.

var bench = {
  openFiles: 0,
  totalOpenFilesEver: 0,
  size: 0,
  nbops: {},    // Number of operations per second (for each file).
};

function profile (param) {
  var data = [
    {doc:'Number of open files', data:bench.openFiles},
    {doc:'Number of opened files ever', data:bench.totalOpenFilesEver},
    {doc:'Core Size (warning -- overstating) (bytes)', data:bench.size},
  ];
  if (param && param.path) {
    data.push({
      doc: 'Number of operations/sec at ' + param.path,
      data: bench.nbops[param.path] || 0,
    });
  }
  return data;
};




// Directory primitives.

// `vpath` is a String identifying a unique virtual path to the file.
// `cb: function ( err, file )`
// is a function triggered when the file is fully constructed.
//
// - the content as a string for text files,
// - an object with a property for each file, that links to the corresponding
//   <File> object, for directories.
//
// This is the constructor for the File object.
//
//     var f = new File ( '/path/to/myfile', function(err, file) {
//       …
//     });
//
// - `f.path` is the path (as a String).
// - `f.meta` contains meta information, like the `type`.
// - `f.isOfType` is a method that takes a mime type (as a String).
// - `f.content` is the data associated to the file.
//   Open the file before using it. Close it when you stop accessing it.
// - `f.open` will count one more user editing that file.
// - `f.close` will count one less user. (That doesn't really matter
//   for folders).
// - `f.rm` will remove the file for good (if nobody is using it).
// - `f.driver` contains additional functionality associated to files of this
//   type.
function File (vpath, cb) {
  this.path = driver.normalize(vpath);
  this.count = 0;   // Number of users modifying the content.
  this.content = null;
  this.meta = {};   // Default: empty (may be overriden by what is on disk).

  this.writo = null;    // Regular write timeout.
  this.nbops = 0;       // Number of operations since last period.

  var that = this;
  driver.loadMeta(this.path, function (err, meta) {
    if (err && err.code !== 'ENOENT' && err.code !== 'ENOTDIR') {
      console.error("While loading metadata:", err.stack);
    } else {
      that.meta = meta || {};
    }

    if (that.meta.type === undefined) {
      // We only need to guess the type if it isn't already in meta.
      type.guessType(that.path, function (err, td) {
        if (err) {
          //console.error("While guessing type", err.stack);
          cb(err);
          return;
        }
        if (td === undefined) that.meta.type = type.fromName['text/plain'];
        else that.meta.type = td;    // `td`: type descriptor (Number).
        that.driver = type.driver(that.meta.type);
        cb(null, that);

        // We had, to guess. Let's write that down.
        driver.dumpMeta(that.path, that.meta);
      });
    } else {
      that.driver = type.driver(that.meta.type);
      cb(null, that);
    }
  });
};



// Checks whether a `file` is of type `typename` (or falls back into it).
//
// - `file`: a File.
// - `mimeType`: a string, case-perfect match of a defined Type.
//
// Warning: this is not *cycle-safe* just yet. This is only an issue if you
// have mistakenly created a loop in the type system.
// It can really screw everything up though.
File.prototype.isOfType = function (mimeType) {
  if (this.meta.type === undefined) {
    console.error('fs:isoftype: file %s has no type.', this.path)
  }
  return type.isCompatible(this.meta.type, type.fromName[mimeType]);
};

// We try to write the file to disk every half minute.
File.writePeriod = 30000;

// The following function is meant to be bound to a file.
// It writes the file to disk if there were modifications.
function writeAtPeriod() {
  bench.nbops[this.path] = this.nbops / File.writePeriod * 1000;
  if (this.nbops > 0) {
    this.write();
  }
  this.nbops = 0;
}

// When you open a file, its content becomes synchronous, immediately available.
// Use it when you need to use readSync and writeSync.
File.prototype.open = function (cb) {
  // If this file had no user, populate the content.
  if (this.count <= 0) {
    var that = this;
    this.driver.read(this.path, function(err, data) {
      if (err !== null) {
        console.error('FS:OPEN:', err.message);
      } else {
        that.content = data;
        // Strings are UCS-2 (or UTF-16): characters are 2 bytes.
        if (data.length)  bench.size += that.content.length * 2;

        // Counts.
        that.count++;
        bench.openFiles++;
        bench.totalOpenFilesEver++;
      }

      if (that.driver.write) {
        that.writo = setInterval(writeAtPeriod.bind(that), File.writePeriod);
        bench.nbops[that.path] = 0;
      }
      cb(err);
    });
  } else {
    // Counts.
    this.count++;
    bench.openFiles++;
    bench.totalOpenFilesEver++;
    cb(null);
  }
};

File.prototype.close = function () {
  if (this.count <= 0) {
    console.error('FS:FILE: count is negative (%s). Keeping it at 0.',
        this.count);
    return;
  }

  this.count--;
  bench.openFiles--;
  if (this.count === 0) {
    var self = this;
    if (this.driver.write) {
      clearInterval(this.writo);
      this.write(function (err) {
        if (err) console.error('FS:WRITE:', err.message);
        self.content = null;    // Let it be garbage-collected.
      });
    } else this.content = null;
    delete fileFromPath[this.path];
  }
};

File.prototype.write = function (cb) {<commit_msg>[log] log file opening and closing<commit_after>/* File system primitives.
 * Copyright © 2011 <NAME>, <NAME>. All rights reserved.
 * The following code is covered by the GPLv2 license. */

var nodefs = require ('fs'),
    nodepath = require ('path'),
    util = require('util'),
    CodeMirrorServer = require('operational-transformation').CodeMirrorServer,
    async = require('async');

var driver = require('./driver'),  // Driver.
    type = require('./type');      // Type system.



// Gather information for the profiler.
//
// Numbers are to computer science what a telescope is to an astrophysicist.

var bench = {
  openFiles: 0,
  totalOpenFilesEver: 0,
  size: 0,
  nbops: {},    // Number of operations per second (for each file).
};

function profile (param) {
  var data = [
    {doc:'Number of open files', data:bench.openFiles},
    {doc:'Number of opened files ever', data:bench.totalOpenFilesEver},
    {doc:'Core Size (warning -- overstating) (bytes)', data:bench.size},
  ];
  if (param && param.path) {
    data.push({
      doc: 'Number of operations/sec at ' + param.path,
      data: bench.nbops[param.path] || 0,
    });
  }
  return data;
};




// Directory primitives.

// `vpath` is a String identifying a unique virtual path to the file.
// `cb: function ( err, file )`
// is a function triggered when the file is fully constructed.
//
// - the content as a string for text files,
// - an object with a property for each file, that links to the corresponding
//   <File> object, for directories.
//
// This is the constructor for the File object.
//
//     var f = new File ( '/path/to/myfile', function(err, file) {
//       …
//     });
//
// - `f.path` is the path (as a String).
// - `f.meta` contains meta information, like the `type`.
// - `f.isOfType` is a method that takes a mime type (as a String).
// - `f.content` is the data associated to the file.
//   Open the file before using it. Close it when you stop accessing it.
// - `f.open` will count one more user editing that file.
// - `f.close` will count one less user. (That doesn't really matter
//   for folders).
// - `f.rm` will remove the file for good (if nobody is using it).
// - `f.driver` contains additional functionality associated to files of this
//   type.
function File (vpath, cb) {
  this.path = driver.normalize(vpath);
  this.count = 0;   // Number of users modifying the content.
  this.content = null;
  this.meta = {};   // Default: empty (may be overriden by what is on disk).

  this.writo = null;    // Regular write timeout.
  this.nbops = 0;       // Number of operations since last period.

  var that = this;
  driver.loadMeta(this.path, function (err, meta) {
    if (err && err.code !== 'ENOENT' && err.code !== 'ENOTDIR') {
      console.error("While loading metadata:", err.stack);
    } else {
      that.meta = meta || {};
    }

    if (that.meta.type === undefined) {
      // We only need to guess the type if it isn't already in meta.
      type.guessType(that.path, function (err, td) {
        if (err) {
          //console.error("While guessing type", err.stack);
          cb(err);
          return;
        }
        if (td === undefined) that.meta.type = type.fromName['text/plain'];
        else that.meta.type = td;    // `td`: type descriptor (Number).
        that.driver = type.driver(that.meta.type);
        cb(null, that);

        // We had, to guess. Let's write that down.
        driver.dumpMeta(that.path, that.meta);
      });
    } else {
      that.driver = type.driver(that.meta.type);
      cb(null, that);
    }
  });
};



// Checks whether a `file` is of type `typename` (or falls back into it).
//
// - `file`: a File.
// - `mimeType`: a string, case-perfect match of a defined Type.
//
// Warning: this is not *cycle-safe* just yet. This is only an issue if you
// have mistakenly created a loop in the type system.
// It can really screw everything up though.
File.prototype.isOfType = function (mimeType) {
  if (this.meta.type === undefined) {
    console.error('fs:isoftype: file %s has no type.', this.path)
  }
  return type.isCompatible(this.meta.type, type.fromName[mimeType]);
};

// We try to write the file to disk every half minute.
File.writePeriod = 30000;

// The following function is meant to be bound to a file.
// It writes the file to disk if there were modifications.
function writeAtPeriod() {
  bench.nbops[this.path] = this.nbops / File.writePeriod * 1000;
  if (this.nbops > 0) {
    this.write();
  }
  this.nbops = 0;
}

// When you open a file, its content becomes synchronous, immediately available.
// Use it when you need to use readSync and writeSync.
File.prototype.open = function (cb) {
  console.log('opening file', this.path);
  // If this file had no user, populate the content.
  if (this.count <= 0) {
    var that = this;
    this.driver.read(this.path, function(err, data) {
      if (err !== null) {
        console.error('FS:OPEN:', err.message);
      } else {
        that.content = data;
        // Strings are UCS-2 (or UTF-16): characters are 2 bytes.
        if (data.length)  bench.size += that.content.length * 2;

        // Counts.
        that.count++;
        bench.openFiles++;
        bench.totalOpenFilesEver++;
      }

      if (that.driver.write) {
        that.writo = setInterval(writeAtPeriod.bind(that), File.writePeriod);
        bench.nbops[that.path] = 0;
      }
      cb(err);
    });
  } else {
    // Counts.
    this.count++;
    bench.openFiles++;
    bench.totalOpenFilesEver++;
    cb(null);
  }
};

File.prototype.close = function () {
  console.log('closing file', this.path);
  if (this.count <= 0) {
    console.error('FS:FILE: count is negative (%s). Keeping it at 0.',
        this.count);
    return;
  }

  this.count--;
  bench.openFiles--;
  if (this.count === 0) {
    var self = this;
    if (this.driver.write) {
      clearInterval(this.writo);
      this.write(function (err) {
        if (err) console.error('FS:WRITE:', err.message);
        self.content = null;    // Let it be garbage-collected.
      });
    } else this.content = null;
    delete fileFromPath[this.path];
  }
};

File.prototype.write = function (cb) {
{'fraction_non_alphanumeric': 0.14433662235994454, 'fraction_numerical': 0.005218951316969747, 'mean_word_length': 2.5100171722953637, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '38473', 'n_tokens_mistral': 3930, 'n_tokens_neox': 3766, 'n_words': 1285}