<commit_before>var async = require('async');

exports.put = put;
exports.del = del;
exports.batch = batch;

function getParentPath(path) {
  if (path === '/') {
    return null;
  }

  var index = path.lastIndexOf('/');

  if (index === 0) {
    return '/';
  }

  return path.slice(0, index);
}

function getPathName(path) {
  if (path === '/') {
    return null;
  }

  return path.slice(path.lastIndexOf('/') + 1);
}

function appendPath(path, name) {
  if (path === '/') {
    return path + name;
  }

  return path + '/' + name;
}

function put(get, path, properties, done) {
  var parentPath = getParentPath(path);
  var name = getPathName(path);

  get(path, onPreviousNode);

  function onPreviousNode(error, node) {
    if (error && error.notFound && parentPath) {
      return get(parentPath, onParentNode);
    }

    if (error && error.notFound) {
      return done(null, [{type: 'put', key: path, value: {children: [], properties: properties}}]);
    }

    if (error) {
      return done(error);
    }

    done(null, [{type: 'put', key: path, value: {children: node.children, properties: properties}}]);
  }

  function onParentNode(error, parent) {
    if (error) {
      return done(error);
    }

    var operations = [
      {
        type: 'put', 
        key: parentPath, 
        value: {
          children: parent.children.concat(name), 
          properties: parent.properties
        }
      },
      {
        type: 'put',
        key: path,
        value: {
          children: [],
          properties: properties
        }
      }
    ];

    done(null, operations);
  }
}

function del(get, path, done) {
  var operations = [];
  var parentPath = getParentPath(path);
  var name = getPathName(path);

  if (parentPath) {
    get(parentPath, onParentNode);
  }
  else {
    iterate(path, onPaths);
  }

  function onParentNode(error, node) {
    if (error) {
      return done(error);
    }

    var children = node.children.filter(isNotDeleted);

    function isNotDeleted(current) {
      return current !== name;
    }

    operations.push({type: 'put', key: parentPath, value: {children: children, properties: node.properties}});

    iterate(path, onPaths);
  }

  function iterate(path, done) {
    get(path, onNode);

    function onNode(error, node) {
      if (error && error.notFound) {
        return done();
      }

      if (error) {
        return done(error);
      }

      operations.push({type: 'del', key: path});

      var children = node.children.map(appendPath.bind(null, path));

      async.each(children, iterate, done);
    }
  }

  function onPaths(error) {
    if (error) {
      return done(error);
    }

    done(null, operations);
  }
}

function batch(get, operations, done) {
  var workspace = {};

  async.mapSeries(operations, computeOperations, onOperationsComputed);

  function computeOperations(operation, done) {
    if (operation.type === 'put') {
      return computePutOperations(operation, done);
    }

    if (operation.type === 'del') {
      return computeDelOperations(operation, done);
    }

    done(new Error('unrecognized operation type'));
  }

  function computePutOperations(operation, done) {
    put(workspaceGet, operation.path, operation.properties, onPutOperations);

    function onPutOperations(error, operations) {
      if (error) {
        return done(error);
      }

      operations.forEach(updateWorkspace);

      done(null, operations);
    }
  }

  function computeDelOperations(operation, done) {
    del(workspaceGet, operation.path, onDelOperations);

    function onDelOperations(error, operations) {
      if (error) {
        return done(error);
      }

      operations.forEach(updateWorkspace);

      done(null, operations);
    }
  }

  function workspaceGet(path, done) {
    var cached = workspace[path];

    if (cached === undefined) {
      return get(path, done);
    }

    if (cached === null) {
      return done({notFound: true});
    }

    done(null, cached);
  }

  function updateWorkspace(operation) {
    if (operation.type === 'put') {
      workspace[operation.key] = operation.value;
    }

    if (operation.type === 'del') {
      workspace[operation.key] = null;
    }
  }

  function onOperationsComputed(error, operations) {
    if (error) {
      return done(error);
    }

    var result = operations.reduce(concat, []);

    function concat(accumulator, current) {
      return accumulator.concat(current);
    }

    done(null, result);
  }
}
<commit_msg>Don't throw an error if the parent of a deleted node doesn't exist<commit_after>var async = require('async');

exports.put = put;
exports.del = del;
exports.batch = batch;

function getParentPath(path) {
  if (path === '/') {
    return null;
  }

  var index = path.lastIndexOf('/');

  if (index === 0) {
    return '/';
  }

  return path.slice(0, index);
}

function getPathName(path) {
  if (path === '/') {
    return null;
  }

  return path.slice(path.lastIndexOf('/') + 1);
}

function appendPath(path, name) {
  if (path === '/') {
    return path + name;
  }

  return path + '/' + name;
}

function put(get, path, properties, done) {
  var parentPath = getParentPath(path);
  var name = getPathName(path);

  get(path, onPreviousNode);

  function onPreviousNode(error, node) {
    if (error && error.notFound && parentPath) {
      return get(parentPath, onParentNode);
    }

    if (error && error.notFound) {
      return done(null, [{type: 'put', key: path, value: {children: [], properties: properties}}]);
    }

    if (error) {
      return done(error);
    }

    done(null, [{type: 'put', key: path, value: {children: node.children, properties: properties}}]);
  }

  function onParentNode(error, parent) {
    if (error) {
      return done(error);
    }

    var operations = [
      {
        type: 'put', 
        key: parentPath, 
        value: {
          children: parent.children.concat(name), 
          properties: parent.properties
        }
      },
      {
        type: 'put',
        key: path,
        value: {
          children: [],
          properties: properties
        }
      }
    ];

    done(null, operations);
  }
}

function del(get, path, done) {
  var operations = [];
  var parentPath = getParentPath(path);
  var name = getPathName(path);

  if (parentPath) {
    get(parentPath, onParentNode);
  }
  else {
    iterate(path, onPaths);
  }

  function onParentNode(error, node) {
    if (error && error.notFound) {
      return done();
    }
    
    if (error) {
      return done(error);
    }

    var children = node.children.filter(isNotDeleted);

    function isNotDeleted(current) {
      return current !== name;
    }

    operations.push({type: 'put', key: parentPath, value: {children: children, properties: node.properties}});

    iterate(path, onPaths);
  }

  function iterate(path, done) {
    get(path, onNode);

    function onNode(error, node) {
      if (error && error.notFound) {
        return done();
      }

      if (error) {
        return done(error);
      }

      operations.push({type: 'del', key: path});

      var children = node.children.map(appendPath.bind(null, path));

      async.each(children, iterate, done);
    }
  }

  function onPaths(error) {
    if (error) {
      return done(error);
    }

    done(null, operations);
  }
}

function batch(get, operations, done) {
  var workspace = {};

  async.mapSeries(operations, computeOperations, onOperationsComputed);

  function computeOperations(operation, done) {
    if (operation.type === 'put') {
      return computePutOperations(operation, done);
    }

    if (operation.type === 'del') {
      return computeDelOperations(operation, done);
    }

    done(new Error('unrecognized operation type'));
  }

  function computePutOperations(operation, done) {
    put(workspaceGet, operation.path, operation.properties, onPutOperations);

    function onPutOperations(error, operations) {
      if (error) {
        return done(error);
      }

      operations.forEach(updateWorkspace);

      done(null, operations);
    }
  }

  function computeDelOperations(operation, done) {
    del(workspaceGet, operation.path, onDelOperations);

    function onDelOperations(error, operations) {
      if (error) {
        return done(error);
      }

      operations.forEach(updateWorkspace);

      done(null, operations);
    }
  }

  function workspaceGet(path, done) {
    var cached = workspace[path];

    if (cached === undefined) {
      return get(path, done);
    }

    if (cached === null) {
      return done({notFound: true});
    }

    done(null, cached);
  }

  function updateWorkspace(operation) {
    if (operation.type === 'put') {
      workspace[operation.key] = operation.value;
    }

    if (operation.type === 'del') {
      workspace[operation.key] = null;
    }
  }

  function onOperationsComputed(error, operations) {
    if (error) {
      return done(error);
    }

    var result = operations.reduce(concat, []);

    function concat(accumulator, current) {
      return accumulator.concat(current);
    }

    done(null, result);
  }
}

{'fraction_non_alphanumeric': 0.14053935540451656, 'fraction_numerical': 0.00065775049331287, 'mean_word_length': 2.650660264105642, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1256430', 'n_tokens_mistral': 2867, 'n_tokens_neox': 2782, 'n_words': 645}