<commit_before>import Schema from '../models/schema'
import Raw from '../serializers/raw'
import warning from '../utils/warning'
import { default as defaultSchema } from '../plugins/schema'

/**
 * Refresh a reference to a node that have been modified in a transform.
 * @param  {Transform} transform
 * @param  {Node} node
 * @return {Node} newNode
 */

function _refreshNode(transform, node) {
  const { state } = transform
  const { document } = state

  if (node.kind == 'document') {
    return document
  }

  return document.getDescendant(node.key)
}

/**
 * Normalize all children of a node
 * @param  {Transform} transform
 * @param  {Schema} schema
 * @param  {Node} node
 * @return {Transform} transform
 */

function _normalizeChildrenWith(transform, schema, node) {
  let { state } = transform

  if (!node.nodes) {
    return transform
  }

  return node.nodes.reduce(
    (t, child) => {
      return t.normalizeNodeWith(schema, child)
    },
    transform
  )
}

/**
 * Normalize a node without its children
 * @param  {Transform} transform
 * @param  {Schema} schema
 * @param  {Node} node
 * @return {Transform} transform
 */

function _normalizeNodeWith(transform, schema, node) {
  let { state } = transform
  const failure = schema.__validate(node)

  // Node is valid?
  if (!failure) {
    return transform
  }

  const { value, rule } = failure

  // Normalize and get the new state
  transform = rule.normalize(transform, node, value)

  // Search for the updated node in the new state
  const newNode = _refreshNode(transform, node)

  // Node no longer exist, go back to normalize parents
  if (!newNode) {
    return transform
  }

  return normalizeNodeWith(transform, schema, newNode)
}

/**
 * Normalize a node (itself and its children) using a schema.
 *
 * @param  {Transform} transform
 * @param  {Schema} schema
 * @param  {Node} node
 * @return {Transform}
 */

export function normalizeNodeWith(transform, schema, node) {
  // Iterate over its children
  transform = _normalizeChildrenWith(transform, schema, node)

  // Refresh the node reference, and normalize it
  node = _refreshNode(transform, node)
  if (node) {
    transform = _normalizeNodeWith(transform, schema, node)
  }

  return transform
}

/**
 * Normalize state using a schema.
 *
 * @param  {Transform} transform
 * @param  {Schema} schema
 * @return {Transform} transform
 */

export function normalizeWith(transform, schema) {
  const { state } = transform
  const { document } = state

  return transform.normalizeNodeWith(schema, document)
}

/**
 * Normalize the state using the core schema.
 *
 * @param  {Transform} transform
 * @return {Transform} transform
 */

export function normalize(transform) {
    return transform
        .normalizeDocument()
        .normalizeSelection()
}

/**
 * Normalize only the document
 *
 * @param  {Transform} transform
 * @return {Transform} transform
 */

export function normalizeDocument(transform) {
    return transform.normalizeWith(defaultSchema)
}

/**
 * Normalize a specific node using core schema
 *
 * @param  {Transform} transform
 * @param  {Node or String} key
 * @return {Transform} transform
 */

export function normalizeNodeByKey(transform, key) {
    const { state } = transform
    const { document } = state
    const node = document.assertDescendant(key)

    return transform.normalizeNodeWith(defaultSchema, node)
}

/**
 * Normalize only the selection.
 *
 * @param  {Transform} transform
 * @return {Transform} transform
 */

export function normalizeSelection(transform) {
  let { state } = transform
  let { document, selection } = state
  selection = selection.normalize(document)

  // If the selection is nulled (not normal)
  if (
    selection.isUnset ||
    !document.hasDescendant(selection.anchorKey) ||
    !document.hasDescendant(selection.focusKey)
  ) {
      warning('Selection was invalid and reset to start of the document')
      const firstText = document.getTexts().first()
      selection = selection.merge({
        anchorKey: firstText.key,
        anchorOffset: 0,
        focusKey: firstText.key,
        focusOffset: 0,
        isBackward: false
      })
  }

  state = state.merge({ selection })
  transform.state = state
  return transform
}
<commit_msg>Fix stupid error in recursive normalize calls<commit_after>import Schema from '../models/schema'
import Raw from '../serializers/raw'
import warning from '../utils/warning'
import { default as defaultSchema } from '../plugins/schema'

/**
 * Refresh a reference to a node that have been modified in a transform.
 * @param  {Transform} transform
 * @param  {Node} node
 * @return {Node} newNode
 */

function _refreshNode(transform, node) {
  const { state } = transform
  const { document } = state

  if (node.kind == 'document') {
    return document
  }

  return document.getDescendant(node.key)
}

/**
 * Normalize all children of a node
 * @param  {Transform} transform
 * @param  {Schema} schema
 * @param  {Node} node
 * @return {Transform} transform
 */

function _normalizeChildrenWith(transform, schema, node) {
  let { state } = transform

  if (!node.nodes) {
    return transform
  }

  return node.nodes.reduce(
    (t, child) => {
      return t.normalizeNodeWith(schema, child)
    },
    transform
  )
}

/**
 * Normalize a node without its children
 * @param  {Transform} transform
 * @param  {Schema} schema
 * @param  {Node} node
 * @return {Transform} transform
 */

function _normalizeNodeWith(transform, schema, node) {
  let { state } = transform
  const failure = schema.__validate(node)

  // Node is valid?
  if (!failure) {
    return transform
  }

  const { value, rule } = failure

  // Normalize and get the new state
  transform = rule.normalize(transform, node, value)

  // Search for the updated node in the new state
  const newNode = _refreshNode(transform, node)

  // Node no longer exist, go back to normalize parents
  if (!newNode) {
    return transform
  }

  return _normalizeNodeWith(transform, schema, newNode)
}

/**
 * Normalize a node (itself and its children) using a schema.
 *
 * @param  {Transform} transform
 * @param  {Schema} schema
 * @param  {Node} node
 * @return {Transform}
 */

export function normalizeNodeWith(transform, schema, node) {
  // Iterate over its children
  transform = _normalizeChildrenWith(transform, schema, node)

  // Refresh the node reference, and normalize it
  node = _refreshNode(transform, node)
  if (node) {
    transform = _normalizeNodeWith(transform, schema, node)
  }

  return transform
}

/**
 * Normalize state using a schema.
 *
 * @param  {Transform} transform
 * @param  {Schema} schema
 * @return {Transform} transform
 */

export function normalizeWith(transform, schema) {
  const { state } = transform
  const { document } = state

  return transform.normalizeNodeWith(schema, document)
}

/**
 * Normalize the state using the core schema.
 *
 * @param  {Transform} transform
 * @return {Transform} transform
 */

export function normalize(transform) {
    return transform
        .normalizeDocument()
        .normalizeSelection()
}

/**
 * Normalize only the document
 *
 * @param  {Transform} transform
 * @return {Transform} transform
 */

export function normalizeDocument(transform) {
    return transform.normalizeWith(defaultSchema)
}

/**
 * Normalize a specific node using core schema
 *
 * @param  {Transform} transform
 * @param  {Node or String} key
 * @return {Transform} transform
 */

export function normalizeNodeByKey(transform, key) {
    const { state } = transform
    const { document } = state
    const node = document.assertDescendant(key)

    return transform.normalizeNodeWith(defaultSchema, node)
}

/**
 * Normalize only the selection.
 *
 * @param  {Transform} transform
 * @return {Transform} transform
 */

export function normalizeSelection(transform) {
  let { state } = transform
  let { document, selection } = state
  selection = selection.normalize(document)

  // If the selection is nulled (not normal)
  if (
    selection.isUnset ||
    !document.hasDescendant(selection.anchorKey) ||
    !document.hasDescendant(selection.focusKey)
  ) {
      warning('Selection was invalid and reset to start of the document')
      const firstText = document.getTexts().first()
      selection = selection.merge({
        anchorKey: firstText.key,
        anchorOffset: 0,
        focusKey: firstText.key,
        focusOffset: 0,
        isBackward: false
      })
  }

  state = state.merge({ selection })
  transform.state = state
  return transform
}

{'fraction_non_alphanumeric': 0.10852531867617822, 'fraction_numerical': 0.0004677815460180096, 'mean_word_length': 3.410520887055183, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4110697', 'n_tokens_mistral': 2466, 'n_tokens_neox': 2410, 'n_words': 776}