<commit_before>import Debug from 'debug'
import invariant from 'tiny-invariant'
import isPlainObject from 'is-plain-object'
import warning from 'tiny-warning'
import { List } from 'immutable'

import CommandsPlugin from '../plugins/commands'
import CorePlugin from '../plugins/core'
import Operation from '../models/operation'
import PathUtils from '../utils/path-utils'
import QueriesPlugin from '../plugins/queries'
import SchemaPlugin from '../plugins/schema'
import Value from '../models/value'

/**
 * Debug.
 *
 * @type {Function}
 */

const debug = Debug('slate:editor')

/**
 * Editor.
 *
 * @type {Editor}
 */

class Editor {
  /**
   * Create a new `Editor` with `attrs`.
   *
   * @param {Object} attrs
   * @param {Object} options
   */

  constructor(attrs = {}, options = {}) {
    const { controller = this, construct = true } = options
    const {
      onChange = () => {},
      plugins = [],
      readOnly = false,
      value = Value.create(),
    } = attrs

    this.controller = controller
    this.middleware = {}
    this.onChange = onChange
    this.operations = List()
    this.readOnly = null
    this.value = null

    this.tmp = {
      dirty: [],
      flushing: false,
      merge: null,
      normalize: true,
      save: true,
    }

    const core = CorePlugin({ plugins })
    registerPlugin(this, core)

    if (construct) {
      this.run('onConstruct')
      this.setReadOnly(readOnly)
      this.setValue(value, options)
    }
  }

  /**
   * Apply an `operation` to the editor, updating its value.
   *
   * @param {Operation|Object} operation
   * @return {Editor}
   */

  applyOperation(operation) {
    const { operations, controller } = this
    let value = this.value

    // Add in the current `value` in case the operation was serialized.
    if (isPlainObject(operation)) {
      operation = { ...operation, value }
    }

    operation = Operation.create(operation)

    // Save the operation into the history. Since `save` is a command, we need
    // to do it without normalizing, since it would have side effects.
    this.withoutNormalizing(() => {
      controller.save(operation)
      value = this.value
    })

    // Apply the operation to the value.
    debug('apply', { operation })
    this.value = operation.apply(value)
    this.operations = operations.push(operation)

    // Get the paths of the affected nodes, and mark them as dirty.
    const newDirtyPaths = getDirtyPaths(operation)

    const dirty = this.tmp.dirty.map(path => {
      path = PathUtils.create(path)
      const transformed = PathUtils.transform(path, operation)
      return transformed.toArray()
    })

    this.tmp.dirty = Array.prototype.concat.apply(newDirtyPaths, dirty)

    // If we're not already, queue the flushing process on the next tick.
    if (!this.tmp.flushing) {
      this.tmp.flushing = true
      Promise.resolve().then(() => this.flush())
    }

    return controller
  }

  /**
   * Flush the editor's current change.
   *
   * @return {Editor}
   */

  flush() {<commit_msg>Deduplicate dirty paths for normalization. (#2916)<commit_after>import Debug from 'debug'
import invariant from 'tiny-invariant'
import isPlainObject from 'is-plain-object'
import warning from 'tiny-warning'
import { List } from 'immutable'

import CommandsPlugin from '../plugins/commands'
import CorePlugin from '../plugins/core'
import Operation from '../models/operation'
import PathUtils from '../utils/path-utils'
import QueriesPlugin from '../plugins/queries'
import SchemaPlugin from '../plugins/schema'
import Value from '../models/value'

/**
 * Debug.
 *
 * @type {Function}
 */

const debug = Debug('slate:editor')

/**
 * Editor.
 *
 * @type {Editor}
 */

class Editor {
  /**
   * Create a new `Editor` with `attrs`.
   *
   * @param {Object} attrs
   * @param {Object} options
   */

  constructor(attrs = {}, options = {}) {
    const { controller = this, construct = true } = options
    const {
      onChange = () => {},
      plugins = [],
      readOnly = false,
      value = Value.create(),
    } = attrs

    this.controller = controller
    this.middleware = {}
    this.onChange = onChange
    this.operations = List()
    this.readOnly = null
    this.value = null

    this.tmp = {
      dirty: [],
      flushing: false,
      merge: null,
      normalize: true,
      save: true,
    }

    const core = CorePlugin({ plugins })
    registerPlugin(this, core)

    if (construct) {
      this.run('onConstruct')
      this.setReadOnly(readOnly)
      this.setValue(value, options)
    }
  }

  /**
   * Apply an `operation` to the editor, updating its value.
   *
   * @param {Operation|Object} operation
   * @return {Editor}
   */

  applyOperation(operation) {
    const { operations, controller } = this
    let value = this.value

    // Add in the current `value` in case the operation was serialized.
    if (isPlainObject(operation)) {
      operation = { ...operation, value }
    }

    operation = Operation.create(operation)

    // Save the operation into the history. Since `save` is a command, we need
    // to do it without normalizing, since it would have side effects.
    this.withoutNormalizing(() => {
      controller.save(operation)
      value = this.value
    })

    // Apply the operation to the value.
    debug('apply', { operation })
    this.value = operation.apply(value)
    this.operations = operations.push(operation)

    // Get the paths of the affected nodes, and mark them as dirty.
    const newDirtyPaths = getDirtyPaths(operation)

    const dirty = this.tmp.dirty.map(path => {
      path = PathUtils.create(path)
      const transformed = PathUtils.transform(path, operation)
      return transformed.toArray()
    })

    const pathIndex = {}
    const dirtyPaths = Array.prototype.concat.apply(newDirtyPaths, dirty)
    this.tmp.dirty = []

    // PERF: De-dupe the paths so we don't do extra normalization.
    dirtyPaths.forEach(dirtyPath => {
      const key = dirtyPath.join(',')

      if (!pathIndex[key]) {
        this.tmp.dirty.push(dirtyPath)
      }

      pathIndex[key] = true
    })

    // If we're not already, queue the flushing process on the next tick.
    if (!this.tmp.flushing) {
      this.tmp.flushing = true
      Promise.resolve().then(() => this.flush())
    }

    return controller
  }

  /**
   * Flush the editor's current change.
   *
   * @return {Editor}
   */

  flush() {
{'fraction_non_alphanumeric': 0.12723874785858902, 'fraction_numerical': 0.0006229559258682448, 'mean_word_length': 2.6323529411764706, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2965568', 'n_tokens_mistral': 1890, 'n_tokens_neox': 1853, 'n_words': 542}