<commit_before>import filter from '../util/filter'
import forEach from '../util/forEach'
import isArrayEqual from '../util/isArrayEqual'
import DocumentIndex from './DocumentIndex'
import ObjectOperation from './data/ObjectOperation'
import DocumentChange from './DocumentChange'
import annotationHelpers from './annotationHelpers'
import { isEntirelySelected } from './selectionHelpers'

/**
  Some helpers for working with Documents.

  @module
  @example

  ```js
  import { documentHelpers } from 'substance'
  documentHelpers.isContainerAnnotation(doc, 'comment')
  ```
*/
export default {
  isContainerAnnotation,
  getPropertyAnnotationsForSelection,
  getContainerAnnotationsForSelection,
  getAnnotationsForSelection,
  getTextForSelection,
  getMarkersForSelection,
  getChangeFromDocument,
  copyNode,
  deleteNode,
  deleteTextRange,
  deleteListRange,
  mergeListItems
}

/**
  @param {Document} doc
  @param {String} type
  @return {Boolean} `true` if given type is a {@link ContainerAnnotation}
*/
function isContainerAnnotation(doc, type) {
  var schema = doc.getSchema();
  return schema.isInstanceOf(type, 'container-annotation');
}

/**
  For a given selection get all property annotations

  @param {Document} doc
  @param {Selection} sel
  @return {PropertyAnnotation[]} An array of property annotations.
          Returns an empty array when selection is a container selection.
*/
function getPropertyAnnotationsForSelection(doc, sel, options) {
  options = options || {};
  if (!sel.isPropertySelection()) {
    return [];
  }
  let path = sel.getPath()
  let annotations = doc.getIndex('annotations').get(path, sel.start.offset, sel.end.offset);
  if (options.type) {
    annotations = filter(annotations, DocumentIndex.filterByType(options.type));
  }
  return annotations;
}

/**
  For a given selection get all container annotations

  @param {Document} doc
  @param {Selection} sel
  @param {String} containerId
  @param {String} options.type provides only annotations of that type
  @return {Array} An array of container annotations
*/
function getContainerAnnotationsForSelection(doc, sel, containerId, options) {
  // ATTENTION: looking for container annotations is not as efficient as property
  // selections, as we do not have an index that has notion of the spatial extend
  // of an annotation. Opposed to that, common annotations are bound
  // to properties which make it easy to lookup.
  if (!containerId) {
    throw new Error("'containerId' is required.");
  }
  options = options || {};
  var index = doc.getIndex('container-annotations');
  var annotations = index.get(containerId, options.type);
  annotations = filter(annotations, function(anno) {
    return sel.overlaps(anno.getSelection());
  });
  return annotations;
}

/**
  For a given selection, get annotations of a certain type

  @param {Document} doc
  @param {Selection} sel
  @param {string} annotationType
  @param {string} containerId (only needed when type is a container annotation)
  @return {array} all matching annotations
*/
function getAnnotationsForSelection(doc, sel, annotationType, containerId) {
  var annos;
  var isContainerAnno = isContainerAnnotation(doc, annotationType);

  if (isContainerAnno) {
    var container = doc.get(containerId, 'strict');
    annos = getContainerAnnotationsForSelection(doc, sel, container, {
      type: annotationType
    });
  } else {
    annos = getPropertyAnnotationsForSelection(doc, sel, { type: annotationType });
  }
  return annos;
}

/**
  For a given selection, get the corresponding text string

  @param {Document} doc
  @param {Selection} sel
  @return {string} text enclosed by the annotation
*/
function getTextForSelection(doc, sel) {
  if (!sel || sel.isNull()) {
    return "";
  } else if (sel.isPropertySelection()) {
    let text = doc.get(sel.start.path);
    return text.substring(sel.start.offset, sel.end.offset);
  } else if (sel.isContainerSelection()) {
    let result = []
    let nodeIds = sel.getNodeIds()
    let L = nodeIds.length
    for (let i = 0; i < L; i++) {
      let id = nodeIds[i]
      let node = doc.get(id)
      if (node.isText()) {
        let text = node.getText()
        if (i === L-1) {
          text = text.slice(0, sel.end.offset)
        }
        if (i === 0) {
          text = text.slice(sel.start.offset)
        }
        result.push(text)
      }
    }
    return result.join('\n');
  }
}

function getMarkersForSelection(doc, sel) {
  // only PropertySelections are supported right now
  if (!sel || !sel.isPropertySelection()) return []
  const path = sel.getPath()
  // markers are stored as one hash for each path, grouped by marker key
  let markers = doc.getIndex('markers').get(path)
  const filtered = filter(markers, function(m) {
    return m.isInsideOf(sel)
  })
  return filtered
}

function getChangeFromDocument(doc) {
  let nodes = doc.getNodes()
  let annotations = []
  let contentNodes = []
  let containers = []

  forEach(nodes, (node) => {
    if (node._isAnnotation) {
      annotations.push(node)
    } else if (node._isContainer) {
      containers.push(node)
    } else {
      contentNodes.push(node)
    }
  })

  let ops = contentNodes.concat(containers).concat(annotations).map((node) => {
    return ObjectOperation.Create([node.id], node.toJSON())
  })

  return new DocumentChange({ops: ops})
}

/*
  Deletes a node and its children and attached annotations
  and removes it from a given container
*/
function deleteNode(doc, node) {
  if (node.isText()) {
    // remove all associated annotations
    let annos = doc.getIndex('annotations').get(node.id)
    for (let i = 0; i < annos.length; i++) {
      doc.delete(annos[i].id);
    }
  }
  // delete recursively
  // ATM we do a cascaded delete if the property has type 'id' or ['array', 'id'] and property 'owned' set,
  // or if it 'file'
  let nodeSchema = node.getSchema()
  forEach(nodeSchema, (prop) => {
    if ((prop.isReference() && prop.isOwned()) || (prop.type === 'file')) {
      if (prop.isArray()) {
        let ids = node[prop.name]
        ids.forEach((id) => {
          deleteNode(doc, doc.get(id))
        })
      } else {
        deleteNode(doc, doc.get(node[prop.name]))
      }
    }
  })
  doc.delete(node.id)
}<commit_msg>Forgiving.<commit_after>import filter from '../util/filter'
import forEach from '../util/forEach'
import isArrayEqual from '../util/isArrayEqual'
import DocumentIndex from './DocumentIndex'
import ObjectOperation from './data/ObjectOperation'
import DocumentChange from './DocumentChange'
import annotationHelpers from './annotationHelpers'
import { isEntirelySelected } from './selectionHelpers'

/**
  Some helpers for working with Documents.

  @module
  @example

  ```js
  import { documentHelpers } from 'substance'
  documentHelpers.isContainerAnnotation(doc, 'comment')
  ```
*/
export default {
  isContainerAnnotation,
  getPropertyAnnotationsForSelection,
  getContainerAnnotationsForSelection,
  getAnnotationsForSelection,
  getTextForSelection,
  getMarkersForSelection,
  getChangeFromDocument,
  copyNode,
  deleteNode,
  deleteTextRange,
  deleteListRange,
  mergeListItems
}

/**
  @param {Document} doc
  @param {String} type
  @return {Boolean} `true` if given type is a {@link ContainerAnnotation}
*/
function isContainerAnnotation(doc, type) {
  var schema = doc.getSchema();
  return schema.isInstanceOf(type, 'container-annotation');
}

/**
  For a given selection get all property annotations

  @param {Document} doc
  @param {Selection} sel
  @return {PropertyAnnotation[]} An array of property annotations.
          Returns an empty array when selection is a container selection.
*/
function getPropertyAnnotationsForSelection(doc, sel, options) {
  options = options || {};
  if (!sel.isPropertySelection()) {
    return [];
  }
  let path = sel.getPath()
  let annotations = doc.getIndex('annotations').get(path, sel.start.offset, sel.end.offset);
  if (options.type) {
    annotations = filter(annotations, DocumentIndex.filterByType(options.type));
  }
  return annotations;
}

/**
  For a given selection get all container annotations

  @param {Document} doc
  @param {Selection} sel
  @param {String} containerId
  @param {String} options.type provides only annotations of that type
  @return {Array} An array of container annotations
*/
function getContainerAnnotationsForSelection(doc, sel, containerId, options) {
  // ATTENTION: looking for container annotations is not as efficient as property
  // selections, as we do not have an index that has notion of the spatial extend
  // of an annotation. Opposed to that, common annotations are bound
  // to properties which make it easy to lookup.
  if (!containerId) {
    throw new Error("'containerId' is required.");
  }
  options = options || {};
  var index = doc.getIndex('container-annotations');
  var annotations = index.get(containerId, options.type);
  annotations = filter(annotations, function(anno) {
    return sel.overlaps(anno.getSelection());
  });
  return annotations;
}

/**
  For a given selection, get annotations of a certain type

  @param {Document} doc
  @param {Selection} sel
  @param {string} annotationType
  @param {string} containerId (only needed when type is a container annotation)
  @return {array} all matching annotations
*/
function getAnnotationsForSelection(doc, sel, annotationType, containerId) {
  var annos;
  var isContainerAnno = isContainerAnnotation(doc, annotationType);

  if (isContainerAnno) {
    var container = doc.get(containerId, 'strict');
    annos = getContainerAnnotationsForSelection(doc, sel, container, {
      type: annotationType
    });
  } else {
    annos = getPropertyAnnotationsForSelection(doc, sel, { type: annotationType });
  }
  return annos;
}

/**
  For a given selection, get the corresponding text string

  @param {Document} doc
  @param {Selection} sel
  @return {string} text enclosed by the annotation
*/
function getTextForSelection(doc, sel) {
  if (!sel || sel.isNull()) {
    return "";
  } else if (sel.isPropertySelection()) {
    let text = doc.get(sel.start.path);
    return text.substring(sel.start.offset, sel.end.offset);
  } else if (sel.isContainerSelection()) {
    let result = []
    let nodeIds = sel.getNodeIds()
    let L = nodeIds.length
    for (let i = 0; i < L; i++) {
      let id = nodeIds[i]
      let node = doc.get(id)
      if (node.isText()) {
        let text = node.getText()
        if (i === L-1) {
          text = text.slice(0, sel.end.offset)
        }
        if (i === 0) {
          text = text.slice(sel.start.offset)
        }
        result.push(text)
      }
    }
    return result.join('\n');
  }
}

function getMarkersForSelection(doc, sel) {
  // only PropertySelections are supported right now
  if (!sel || !sel.isPropertySelection()) return []
  const path = sel.getPath()
  // markers are stored as one hash for each path, grouped by marker key
  let markers = doc.getIndex('markers').get(path)
  const filtered = filter(markers, function(m) {
    return m.isInsideOf(sel)
  })
  return filtered
}

function getChangeFromDocument(doc) {
  let nodes = doc.getNodes()
  let annotations = []
  let contentNodes = []
  let containers = []

  forEach(nodes, (node) => {
    if (node._isAnnotation) {
      annotations.push(node)
    } else if (node._isContainer) {
      containers.push(node)
    } else {
      contentNodes.push(node)
    }
  })

  let ops = contentNodes.concat(containers).concat(annotations).map((node) => {
    return ObjectOperation.Create([node.id], node.toJSON())
  })

  return new DocumentChange({ops: ops})
}

/*
  Deletes a node and its children and attached annotations
  and removes it from a given container
*/
function deleteNode(doc, node) {
  if (!node) {
    console.warn('Invalid arguments')
    return
  }
  if (node.isText()) {
    // remove all associated annotations
    let annos = doc.getIndex('annotations').get(node.id)
    for (let i = 0; i < annos.length; i++) {
      doc.delete(annos[i].id);
    }
  }
  // delete recursively
  // ATM we do a cascaded delete if the property has type 'id' or ['array', 'id'] and property 'owned' set,
  // or if it 'file'
  let nodeSchema = node.getSchema()
  forEach(nodeSchema, (prop) => {
    if ((prop.isReference() && prop.isOwned()) || (prop.type === 'file')) {
      if (prop.isArray()) {
        let ids = node[prop.name]
        ids.forEach((id) => {
          deleteNode(doc, doc.get(id))
        })
      } else {
        deleteNode(doc, doc.get(node[prop.name]))
      }
    }
  })
  doc.delete(node.id)
}
{'fraction_non_alphanumeric': 0.12190854870775347, 'fraction_numerical': 0.0007952286282306163, 'mean_word_length': 3.494639027877055, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6505646', 'n_tokens_mistral': 3779, 'n_tokens_neox': 3682, 'n_words': 1101}