<commit_before>var cp = require('child_process');
var fs = require('fs');
var path = require('path');
var _ = require('underscore');

/***** Indexing *****/

/**
* Recursively traverse files and directories invoking the callback
* function on each element found.
*/
explore_recursive = function(dir, callback, context, base) {
  if( base==undefined ) base = path.basename(dir);
  _.map(fs.readdirSync(dir), function(file) {
    var f = path.join(dir, file); // this is the real file name on disk
    var n = path.join(base, file); // this is the file name in the archive
    var s = fs.lstatSync(f);
    if( s.isDirectory() ) {
      context = callback(n, 'D', f, context);
      explore_recursive(f, callback, context, n);
    } else if( s.isFile() ) {
      context = callback(n, 'F', f, context);
    } else {
      // TODO: handle symbolic links and special files
      console.error("Error: File type not supported, ignoring: "+n);
    }
  });
  return context;
};

/**
* Return file stats (see return statement).
*/
get_element = function(file, full_name) {
  if( full_name==undefined ) full_name = file;
  var s = fs.lstatSync(file), hash;
  if( s.isFile() ) {
    // hashes are only computed for files
    // FIXME: make a better helper function
    var h = cp.spawnSync('md5',['-r',file]);
    if( h.status!=0 ) {
      throw new Error(file+': '+h.error);
    }
    hash = String(h.stdout).split(' ')[0];
  }
  return {
    full_name: full_name,
    base_name: path.basename(file),
    type: (s.isDirectory()?'D':(s.isSymbolicLink()?'L':(s.isFile()?'F':'O'))),
    size: s.size,
    mode: s.mode,
    hash: hash,
  };
};

/**
* Construct an index of files for the specified directory.
*/
construct_file_index = function(dir, base) {
  return explore_recursive(dir, function(name, type, file, index) {
    index[name] = get_element(file, name);
    return index;
  }, {}, base);
};

/**
* Construct an index of objects from the specified file index.
*/
construct_object_index = function(file_index) {
  var object_index = {};
  _.map(_.values(file_index), function(element) {
    if( element.type=='F')
    {
      // hashes are only computed for files
      if( _.has(object_index, element.hash) ) {
        object_index[element.hash].refs += 1;
        object_index[element.hash].files.push(element);
      } else {
        object_index[element.hash] = {
          refs: 1,
          files: [element],
        };
      }
    }
  });
  return object_index;
}

/**
* Construct a full index of files and objects.
*/
construct_index = function(file_index) {
  return {
    files: file_index,
    objects: construct_object_index(file_index)
  };
};

/***** Handling changes *****/

/**
* Dummy callback function which only print information to stdout.
*/
var dummy_callback = {
  new: function(new_element) { console.info('new: '+new_element.full_name); },
  rename: function(old_element,new_element) { console.info('rename: '+old_element.full_name+'->'+new_element.full_name); },
  copy: function(old_element,new_element) { console.info('copy: '+old_element.full_name+'->'+new_element.full_name); },
  modify: function(old_element,new_element) { console.info('modify: '+new_element.full_name+' ('+old_element.size+' bytes -> '+new_element.size+' bytes)'); },
  modify_instance: function(old_element,new_element) { console.info('modify one instance: '+new_element.full_name+' ('+old_element.size+' bytes -> '+new_element.size+' bytes)'); },
  chmod: function(old_element,new_element) { console.info('chmod: '+new_element.full_name+' ('+old_element.mode+'->'+new_element.mode+')'); },
  unchanged: function(old_element,new_element) { console.info('no change: '+new_element.full_name); },
  delete: function(old_element) { console.info('delete: '+old_element.full_name); },
  delete_instance: function(old_element) { console.info('delete one instance: '+old_element.full_name); },
};

/**
* Detect changes and launch the appropriate callback actions.
* This is the core function of arcsavvy!
*/
compute_changes = function(start_index, end_file_index, end_element, callback) {
  if( callback==undefined ) callback = dummy_callback;
  if( start_index.seen==undefined ) start_index.seen = {};

  var start_element = start_index.files[end_element.full_name];
    // not found in end index
  if( end_file_index && end_file_index[end_element.full_name]==undefined ) {
    var end_index = construct_index(end_file_index);
    var end_element2 = end_index.objects[end_element.hash];
    if( end_element2==undefined ) {
      // hash not found
      callback.delete(end_element);
    } else {
      if( end_element2.refs==1 ) {
        callback.rename(end_element,end_element2.files[0]);
      } else {
        console.warn("Warning: Ambiguous rename, I consider the file as being copy of: "+end_element2.files[0].full_name);
        callback.rename(end_element,end_element2.files[0]);
      }
    }
  } else if( start_element==undefined ) {
    // file not found in index
    start_element = start_index.objects[end_element.hash];
    if( start_element==undefined ) {
      // hash not found
      callback.new(end_element);
    } else {
      // hash found
      if( start_element.refs==1 ) {
        start_element = start_element.files[0];
        // if end_file_index is available, then forward detection is possible
        // so we can directly rename the file, otherwise we will have to
        // first make a copy of it, and delete it later
        if( end_file_index && !_.has(end_file_index, start_element.full_name) ) {
          // file is not present anymore
          start_index.seen[start_element.hash] = true;
          callback.rename(start_element,end_element);
        } else {
          // file is still there
          callback.copy(start_element,end_element);
        }
      } else {
        // FIXME: what is the appropriate behavior?

        // there are multiple references, consider as new and emit a warning
        // console.log("Warning: Ambiguous rename, I consider the file as being new")
        // callback.new(end_element);

        // there are multiple references, consider arbitrarily as a copy of first one
        start_element = start_element.files[0];<commit_msg>documentation for compute_changes<commit_after>var cp = require('child_process');
var fs = require('fs');
var path = require('path');
var _ = require('underscore');

/***** Indexing *****/

/**
* Recursively traverse files and directories invoking the callback
* function on each element found.
*/
explore_recursive = function(dir, callback, context, base) {
  if( base==undefined ) base = path.basename(dir);
  _.map(fs.readdirSync(dir), function(file) {
    var f = path.join(dir, file); // this is the real file name on disk
    var n = path.join(base, file); // this is the file name in the archive
    var s = fs.lstatSync(f);
    if( s.isDirectory() ) {
      context = callback(n, 'D', f, context);
      explore_recursive(f, callback, context, n);
    } else if( s.isFile() ) {
      context = callback(n, 'F', f, context);
    } else {
      // TODO: handle symbolic links and special files
      console.error("Error: File type not supported, ignoring: "+n);
    }
  });
  return context;
};

/**
* Return file stats (see return statement).
*/
get_element = function(file, full_name) {
  if( full_name==undefined ) full_name = file;
  var s = fs.lstatSync(file), hash;
  if( s.isFile() ) {
    // hashes are only computed for files
    // FIXME: make a better helper function
    var h = cp.spawnSync('md5',['-r',file]);
    if( h.status!=0 ) {
      throw new Error(file+': '+h.error);
    }
    hash = String(h.stdout).split(' ')[0];
  }
  return {
    full_name: full_name,
    base_name: path.basename(file),
    type: (s.isDirectory()?'D':(s.isSymbolicLink()?'L':(s.isFile()?'F':'O'))),
    size: s.size,
    mode: s.mode,
    hash: hash,
  };
};

/**
* Construct an index of files for the specified directory.
*/
construct_file_index = function(dir, base) {
  return explore_recursive(dir, function(name, type, file, index) {
    index[name] = get_element(file, name);
    return index;
  }, {}, base);
};

/**
* Construct an index of objects from the specified file index.
*/
construct_object_index = function(file_index) {
  var object_index = {};
  _.map(_.values(file_index), function(element) {
    if( element.type=='F')
    {
      // hashes are only computed for files
      if( _.has(object_index, element.hash) ) {
        object_index[element.hash].refs += 1;
        object_index[element.hash].files.push(element);
      } else {
        object_index[element.hash] = {
          refs: 1,
          files: [element],
        };
      }
    }
  });
  return object_index;
}

/**
* Construct a full index of files and objects.
*/
construct_index = function(file_index) {
  return {
    files: file_index,
    objects: construct_object_index(file_index)
  };
};

/***** Handling changes *****/

/**
* Dummy callback function which only print information to stdout.
*/
var dummy_callback = {
  new: function(new_element) { console.info('new: '+new_element.full_name); },
  rename: function(old_element,new_element) { console.info('rename: '+old_element.full_name+'->'+new_element.full_name); },
  copy: function(old_element,new_element) { console.info('copy: '+old_element.full_name+'->'+new_element.full_name); },
  modify: function(old_element,new_element) { console.info('modify: '+new_element.full_name+' ('+old_element.size+' bytes -> '+new_element.size+' bytes)'); },
  modify_instance: function(old_element,new_element) { console.info('modify one instance: '+new_element.full_name+' ('+old_element.size+' bytes -> '+new_element.size+' bytes)'); },
  chmod: function(old_element,new_element) { console.info('chmod: '+new_element.full_name+' ('+old_element.mode+'->'+new_element.mode+')'); },
  unchanged: function(old_element,new_element) { console.info('no change: '+new_element.full_name); },
  delete: function(old_element) { console.info('delete: '+old_element.full_name); },
  delete_instance: function(old_element) { console.info('delete one instance: '+old_element.full_name); },
};

/**
* Detect changes and launch the appropriate callback actions.
* This is the core function of arcsavvy!
*/
compute_changes = function(start_index, end_file_index, end_element, callback) {
  if( callback==undefined ) callback = dummy_callback;
  if( start_index.seen==undefined ) start_index.seen = {};

  var start_element = start_index.files[end_element.full_name];
  if( end_file_index && end_file_index[end_element.full_name]==undefined ) {
    // not found in end index: deleted or renamed?
    var end_index = construct_index(end_file_index);
    var end_element2 = end_index.objects[end_element.hash];
    if( end_element2==undefined ) {
      // hash not found: deleted
      callback.delete(end_element);
    } else {
      // hash found: renamed
      if( end_element2.refs==1 ) {
        callback.rename(end_element,end_element2.files[0]);
      } else {
        console.warn("Warning: Ambiguous rename, I consider the file as being copy of: "+end_element2.files[0].full_name);
        callback.rename(end_element,end_element2.files[0]);
      }
    }
  } else if( start_element==undefined ) {
    // file not found in start index: new, renamed or copied?
    start_element = start_index.objects[end_element.hash];
    if( start_element==undefined ) {
      // hash not found: new
      callback.new(end_element);
    } else {
      // hash found: renamed or copied?
      if( start_element.refs==1 ) {
        start_element = start_element.files[0];
        // if end_file_index is available, then forward detection is possible
        // so we can directly rename the file, otherwise we will have to
        // first make a copy of it, and delete it later
        if( end_file_index && !_.has(end_file_index, start_element.full_name) ) {
          // file is not present anymore: renamed
          start_index.seen[start_element.hash] = true;
          callback.rename(start_element,end_element);
        } else {
          // file is still there: copied
          callback.copy(start_element,end_element);
        }
      } else {
        // FIXME: what is the appropriate behavior?
        // FIXME: can it be a rename? can end_file_index help us find out?

        // there are multiple references, consider as new and emit a warning
        // console.log("Warning: Ambiguous rename, I consider the file as being new")
        // callback.new(end_element);

        // there are multiple references, consider arbitrarily as a copy of first one
        start_element = start_element.files[0];
{'fraction_non_alphanumeric': 0.14034671099501306, 'fraction_numerical': 0.002849679411066255, 'mean_word_length': 3.4438972915933874, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '25657', 'n_tokens_mistral': 3974, 'n_tokens_neox': 3932, 'n_words': 987}