<commit_before>//operations

//retain {op:'retain', n:5}
//insert {op:'insert', values:[a b c]}
//insert {op:'insert', values:"text"}
//delete {op:'delete', values:[a b c]}
//delete {op:'delete', values:"text"}
// unpush, unpop

//with linearisation we also need UP and DOWN (with kind)

//with critical regions we also need START END

var Region = require('./region');
var Selection = require('./selection');
var List = require('./list');
var AttributedString = require('./string');
var sym = require('./symbol').sym;
var equal = require('./is').equal;

var parse = require('./parser');

//op types
var RETAIN  = "retain";
var INSERT  = "insert";
var DELETE  = "delete";
var START   = "start";
var END     = "end";

//value types
var SYMBOL  = "sym";
var CHAR    = "char";
var OBJ     = "obj";
var PUSH    = "push";
var POP     = "pop";

//push types
var STRING  = "string";
var LIST    = "list";

//linearisation
var pushA = function(attributes) { return {op:INSERT, value:LIST, type:PUSH, n:1, attributes:attributes}; }
var pushS = function(attributes) { return {op:INSERT, value:STRING, type:PUSH, n:1, attributes:attributes}; }
var unpushA = function(attributes) { return {op:DELETE, value:LIST, type:PUSH, n:1, unattributes:attributes}; }
var unpushS = function(attributes) { return {op:DELETE, value:STRING, type:PUSH, n:1, unattributes:attributes}; }
var pop   = {op:INSERT, type:POP, n:1};
var unpop = {op:DELETE, type:POP, n:1};

//critical regions
var start   = {op:START};
var end     = {op:END};

//standard text/list operations
function r(n, attributes, unattributes) {
  return {op:RETAIN, n:n, attributes:attributes, unattributes:unattributes};
}

function _i(vs, type, attributes) {
  return {op:INSERT,
    value:vs,
    type:type,
    n:(type === CHAR ? vs.length : 1),
    attributes:attributes
  };
}

function _d(vs, type, attributes) {
  return {op:DELETE,
    value:vs,
    type:type,
    n:(type === CHAR ? vs.length : 1),
    unattributes:attributes //we should make this attributes or 
                            //unattributes depending on what makes 
                            //the code simplest
  };
}

function _trim(ops) {
  i = ops.length-1;
  while (i >= 0) {
    var op = ops[i];
    if (op.op !== RETAIN || op.attributes || op.unattributes)
      break;
    ops.pop();
    i--;
  }
  return ops;
}

//RULES
// [r(a),r(b)] -> [r(a+b)] iff attributes match
// [i(a),i(b)] -> [i(a.concat(b))] iff char and attributes match
// [start,end] -> []
// [end,start] -> []

function _push(ops, op) {
  if (ops.length === 0)
    return ops.push(op);

  var t = ops[ops.length-1];
  switch (op.op) {
    case RETAIN:
      if (t.op === RETAIN && equal(op.attributes, t.attributes) && equal(op.unattributes, t.unattributes)) {
        ops.pop();
        return _push(ops, r(t.n + op.n, op.attributes, op.unattributes));
      }
      break;
    case INSERT:
      if (t.op === INSERT &&
        op.type === CHAR &&
        t.type === CHAR &&
        equal(op.attributes, t.attributes)) {
        ops.pop();
        return _push(ops, i(t.value.concat(op.values)), CHAR, t.attributes);
      }
      break;
    case START:
      if (t.op === END)
        return ops.pop();
      break;
    case END:
      if (t.op === START)
        return ops.pop();
      break;
  }
  return ops.push(op);
}

/*
takes a source and target document and returns operations to modify
the source document into the target document.

Note: assumes simple operations where objects stay at the same level
within the tree.<commit_msg>fixed merge of insert<commit_after>//operations

//retain {op:'retain', n:5}
//insert {op:'insert', values:[a b c]}
//insert {op:'insert', values:"text"}
//delete {op:'delete', values:[a b c]}
//delete {op:'delete', values:"text"}
// unpush, unpop

//with linearisation we also need UP and DOWN (with kind)

//with critical regions we also need START END

var Region = require('./region');
var Selection = require('./selection');
var List = require('./list');
var AttributedString = require('./string');
var sym = require('./symbol').sym;
var equal = require('./is').equal;

var parse = require('./parser');

//op types
var RETAIN  = "retain";
var INSERT  = "insert";
var DELETE  = "delete";
var START   = "start";
var END     = "end";

//value types
var SYMBOL  = "sym";
var CHAR    = "char";
var OBJ     = "obj";
var PUSH    = "push";
var POP     = "pop";

//push types
var STRING  = "string";
var LIST    = "list";

//linearisation
var pushA = function(attributes) { return {op:INSERT, value:LIST, type:PUSH, n:1, attributes:attributes}; }
var pushS = function(attributes) { return {op:INSERT, value:STRING, type:PUSH, n:1, attributes:attributes}; }
var unpushA = function(attributes) { return {op:DELETE, value:LIST, type:PUSH, n:1, unattributes:attributes}; }
var unpushS = function(attributes) { return {op:DELETE, value:STRING, type:PUSH, n:1, unattributes:attributes}; }
var pop   = {op:INSERT, type:POP, n:1};
var unpop = {op:DELETE, type:POP, n:1};

//critical regions
var start   = {op:START};
var end     = {op:END};

//standard text/list operations
function r(n, attributes, unattributes) {
  return {op:RETAIN, n:n, attributes:attributes, unattributes:unattributes};
}

function _i(vs, type, attributes) {
  return {op:INSERT,
    value:vs,
    type:type,
    n:(type === CHAR ? vs.length : 1),
    attributes:attributes
  };
}

function _d(vs, type, attributes) {
  return {op:DELETE,
    value:vs,
    type:type,
    n:(type === CHAR ? vs.length : 1),
    unattributes:attributes //we should make this attributes or 
                            //unattributes depending on what makes 
                            //the code simplest
  };
}

function _trim(ops) {
  i = ops.length-1;
  while (i >= 0) {
    var op = ops[i];
    if (op.op !== RETAIN || op.attributes || op.unattributes)
      break;
    ops.pop();
    i--;
  }
  return ops;
}

//RULES
// [r(a),r(b)] -> [r(a+b)] iff attributes match
// [i(a),i(b)] -> [i(a.concat(b))] iff char and attributes match
// [start,end] -> []
// [end,start] -> []

function _push(ops, op) {
  if (ops.length === 0)
    return ops.push(op);

  var t = ops[ops.length-1];
  switch (op.op) {
    case RETAIN:
      if (t.op === RETAIN && equal(op.attributes, t.attributes) && equal(op.unattributes, t.unattributes)) {
        ops.pop();
        return _push(ops, r(t.n + op.n, op.attributes, op.unattributes));
      }
      break;
    case INSERT:
      if (t.op === INSERT &&
        op.type === CHAR &&
        t.type === CHAR &&
        equal(op.attributes, t.attributes)) {
        ops.pop();
        return _push(ops, _i(t.value.concat(op.values)), CHAR, t.attributes);
      }
      break;
    case START:
      if (t.op === END)
        return ops.pop();
      break;
    case END:
      if (t.op === START)
        return ops.pop();
      break;
  }
  return ops.push(op);
}

/*
takes a source and target document and returns operations to modify
the source document into the target document.

Note: assumes simple operations where objects stay at the same level
within the tree.
{'fraction_non_alphanumeric': 0.16843295181574114, 'fraction_numerical': 0.0036738731100748904, 'mean_word_length': 2.9541899441340784, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7511754', 'n_tokens_mistral': 2408, 'n_tokens_neox': 2335, 'n_words': 688}