<commit_before>import _ from 'lodash'
import invariant from 'react/lib/invariant'

import alt from '../core/alt'
import {ATTR, attributesEqual} from '../core/attributes'

import EditorActions from './EditorActions'
import { BASE_CHAR, EOF } from '../core/RichText'
import { pushArray/*, logInGroup*/ } from '../core/utils'
import { default as tokenizer, isWhitespace } from '../core/tokenizer'
import writeHtml from '../core/htmlwriter'
import writeText from '../core/textwriter'
import TextFontMetrics from '../core/TextFontMetrics'
import { charEq, charArrayEq, charId, Line, lineContainingChar } from '../core/EditorCommon'

const FLOW_DEBUG = false
const ACTION_DELETE = 'delete'
const ACTION_INSERT = 'insert'
const ACTION_ATTRIBUTES = 'attributes'

/**
 * The EditorStore is the editor's single source of truth for application state and logic related to the editor.
 * View state updates are provided to the view components in setState callbacks. This uses the Facebook Flux
 * unidirectional flow idea: actions (from the view and elsewhere) -> dispatcher -> store -> view.
 *
 * See https://facebook.github.io/flux/docs/overview.html
 */
class EditorStore {
  constructor() {
    this.exportPublicMethods({
      getSelection: () => this.getSelection(),
      getSelectionRich: () => this.getSelectionRich(),
      getSelectionHtml: () => this.getSelectionHtml(),
      getSelectionText: () => this.getSelectionText()
    })

    this.bindActions(EditorActions)

    this.config = null
    this.replica = null

    this.state = {
      position: BASE_CHAR,
      positionEolStart: true,
      cursorMotion: false,
      selectionActive: false,
      remoteCursors: {},
      remoteNameReveal: [],
      focus: false,
      loaded: false,
      lines: []
    }
  }

  initialize({config, replica}) {
    TextFontMetrics.setConfig(config)
    this.config = config
    this.replica = replica

    this.setState({focus: config.initialFocus})
  }

  initializeCursorModel({cursorSet, cursorModel}) {
    this.cursorSet = cursorSet
    this.cursorModel = cursorModel
    this._setRemoteCursorModel()
  }

  replicaInitialized() {
    this.setState({loaded: true})
    this._flow()
  }

  replicaUpdated() {
    // TODO can we determine what changed and do a more focused flow via the modifications parameter?
    this._flow()
  }

  focusInput() {
    this._delayedCursorBlink(0)
    this.setState({focus: true})
  }

  inputFocusLost() {
    this.setState({focus: false})
  }

  setRemoteCursorPosition(remoteCursor) {
    let id = remoteCursor.model._id
    // some bug in Swarm.js: sometimes the id is blank if a lot of cursor events have happened quickly in succession
    if(!id) return
    let remoteCursors = this.state.remoteCursors
    remoteCursors[id] = remoteCursor
    this.setState({remoteCursors: remoteCursors})
    this._delayedRemoteCursorNameReveal(id)
  }

  unsetRemoteCursorPosition(remoteCursor) {
    let id = remoteCursor.model._id
    // some bug in Swarm.js: sometimes the id is blank if a lot of cursor events have happened quickly in succession
    if(!id) return
    let remoteCursors = this.state.remoteCursors
    if(id in remoteCursors) {
      delete remoteCursors[id]
      this.setState({remoteCursors: remoteCursors})
    }
  }

  updateRemoteCursorModel() {
    this._setRemoteCursorModel()
  }

  revealRemoteCursorName(remoteCursor) {
    this._delayedRemoteCursorNameReveal(remoteCursor.model._id)
  }

  navigateLeft() {
    this._navigateLeftRight(-1)
  }

  navigateRight() {
    this._navigateLeftRight(1)
  }

  navigateUp() {
    this._navigateUpDown(-1)
  }

  navigateDown() {
    this._navigateUpDown(1)
  }

  navigatePageUp() {
    // assume for now a page is 10 lines
    this._navigateUpDown(-10)
  }

  navigatePageDown() {
    // assume for now a page is 10 lines
    this._navigateUpDown(10)
  }

  navigateStart() {
    this._resetPosition()
  }

  navigateStartLine() {
    if(this._emptyEditor()) {
      this._resetPosition()
      return
    }

    let {line} = lineContainingChar(this.state.lines, this.state.position, this.state.positionEolStart)
    this._setPosition(line.start, true)
  }

  navigateEnd() {
    if(this._emptyEditor()) {
      this._resetPosition()
      return
    }

    let positionEolStart = false
    if(this._lastLine().isEof()) {
      positionEolStart = true
    }
    this._setPosition(this._relativeChar(BASE_CHAR, -1), positionEolStart)
  }

  navigateEndLine() {
    if(this._emptyEditor()) {
      this._resetPosition()
      return
    }

    let {line} = lineContainingChar(this.state.lines, this.state.position, this.state.positionEolStart)
    let position
    let positionEolStart = false
    if(line.isEof() || line.chunks.length === 0) {
      position = this.state.position
      positionEolStart = true
    } else if (line.isHard()) {
      position = this._relativeChar(line.end, -1, 'limit')
    } else {
      position = line.end
    }
    this._setPosition(position, positionEolStart)
  }

  navigateWordLeft() {
    this._navigateWordLeftRight(-1)
  }

  navigateWordRight() {
    this._navigateWordLeftRight(1)
  }

  navigateToCoordinates(coordinates) {
    let {position, positionEolStart} = this._coordinatesToPosition(coordinates)

    // set the position and selection anchor if the user continues the selection later
    position = position ? position : BASE_CHAR

    this._setPosition(position, positionEolStart)
    this.setState({
      selectionAnchorChar: position
    })
  }

  selectionLeft() {
    this._selectionLeftRight(-1)
  }

  selectionRight() {
    this._selectionLeftRight(1)
  }

  selectionUp() {
    this._selectionUpDown(-1)
  }

  selectionDown() {
    this._selectionUpDown(1)
  }

  selectionPageUp() {
    // assume for now a page is 10 lines
    this._selectionUpDown(-10)
  }

  selectionPageDown() {
    // assume for now a page is 10 lines
    this._selectionUpDown(10)
  }

  selectionStart() {
    this._modifySelection(BASE_CHAR, true)
  }

  selectionStartLine() {
    let {line} = lineContainingChar(this.state.lines, this.state.position, this.state.positionEolStart)
    this._modifySelection(line.start, true)
  }

  selectionEnd() {
    let toChar = this._lastLine() && this._lastLine().isEof() ? EOF : this._relativeChar(BASE_CHAR, -1)
    this._modifySelection(toChar, toChar === EOF)
  }

  /**
   * Google docs behavior (as of 2015-06-22) is:
   * - line with soft return, select line with space at end
   * - line with hard return, select line without selecting the hard return (no action on empty lines)
   * - EOF line, no action
   *
   * Word 2010 behavior is:
   * - line with soft return, select line with space at end
   * - line with hard return, select line including the hard return (same on empty lines)
   * - EOF line, show "space" selection at EOF containing a newline
   *
   * We implement the Google docs behavior here, which seems a bit more intuitive.
   */
  selectionEndLine() {
    let {line} = lineContainingChar(this.state.lines, this.state.position, this.state.positionEolStart)
    let toChar
    let positionEolStart = false
    if(line.isEof()) {
      toChar = this.state.position
      positionEolStart = true
    } else if (line.isHard()) {
      toChar = this._relativeChar(line.end, -1, 'limit')
    } else {
      toChar = line.end
    }
    this._modifySelection(toChar, positionEolStart)
  }

  selectionWordLeft() {
    let position
    let positionEolStart
    if(this.state.position === EOF) {
      position = this._lastLine().start
      positionEolStart = true
    } else {
      position = this._wordStartRelativeTo(this.state.position)
      positionEolStart = !lineContainingChar(this.state.lines, this.state.position).endOfLine
    }
    this._modifySelection(position, positionEolStart)
  }

  selectionWordRight() {
    let position = this._wordEndRelativeTo(this.state.position)
    let endOfLine = lineContainingChar(this.state.lines, this.state.position).endOfLine

    this._modifySelection(position, !endOfLine)
  }

  selectionAll() {
    this._setPosition(BASE_CHAR)
    if(this.state.lines.length === 0) {
      this._modifySelection(EOF, true)
    } else {
      let lastChar = this._lastLine() && this._lastLine().isEof() ? EOF : this._lastLine().end
      this._modifySelection(lastChar, false)
    }
  }

  selectToCoordinates(coordinates) {
    let {position, positionEolStart} = this._coordinatesToPosition(coordinates)
    this._modifySelection(position, positionEolStart)
  }

  /**
   * Word selection follows Google Docs and Microsoft Word 2010 behavior of:
   * - selecting the word that was clicked, and one or more spaces following it
   * - if clicking at the end of a line with a soft return, the first word on the next line is selected
   * - if clicking at the end of a line with a hard return, the hard return is selected (shown as a "space")
   * - if clicking at the end of the last (empty) line, a (non-existent) hard return at EOF is selected (shown as
   *   a "space")
   */
  selectWordAtCurrentPosition() {
    if(this.state.lines.length === 0
      || lineContainingChar(this.state.lines, this.state.position, true).line.isEof()) {
      this._setPosition(this.state.position)
      this._modifySelection(EOF, true)
    } else {
      let word = this._wordRelativeTo(this.state.position)
      this._setPosition(word.start)
      this._modifySelection(word.end, false)
    }
  }

  copySelection(copyHandler) {
    let selectionChunks = this._getSelectionRich()
    if(selectionChunks && selectionChunks.length > 0) {
      let copiedText = writeText(selectionChunks)
      let copiedHtml = writeHtml(selectionChunks)
      copyHandler(copiedText, copiedHtml, selectionChunks)
    }
  }

  getSelection() {
    return this.replica.getTextRange(this.state.selectionLeftChar, this.state.selectionRightChar)
  }

  getSelectionRich() {
    return this._getSelectionRich()
  }

  getSelectionHtml() {
    return writeHtml(this._getSelectionRich())
  }

  getSelectionText() {
    return writeText(this._getSelectionRich())
  }

  insertChars({value, attributes, atPosition, reflow}) {
    return this._insertChars(value, attributes, atPosition, reflow)
  }

  insertCharsBatch(chunks) {
    let firstInsertPosition = this.state.position
    let insertPosition = null
    chunks.forEach(c => {
      insertPosition = this._insertChars(c.text, c.attrs || {}, insertPosition, false)
    })
    this._flow({start: firstInsertPosition, end: insertPosition, action: 'insert'})
  }

  eraseCharBack() {
    if(this.state.selectionActive) {
      this._eraseSelection()
    } else if(!charEq(this.state.position, BASE_CHAR)) {
      let position = this._relativeChar(this.state.position, -1)
      this.replica.rmChars(this.state.position)
      this._flow({ start: position, end: this.state.position, action: ACTION_DELETE})

      let endOfLine = lineContainingChar(this.state.lines, position).endOfLine
      this._setPosition(position, endOfLine)
    }
  }

  eraseCharForward() {
    if(this.state.selectionActive) {
      this._eraseSelection()
    } else if(!this._cursorAtEnd()) {
      let next = this._relativeChar(this.state.position, 1, 'limit')
      this.replica.rmChars(next)
      this._flow({ start: this.state.position, end: next, action: ACTION_DELETE})

      let endOfLine = lineContainingChar(this.state.lines, this.state.position).endOfLine
      this._setPosition(this.state.position, endOfLine)
    }
  }

  eraseWordBack() {
    if(this.state.selectionActive) {
      this._eraseSelection()
    } else {
      let position = this.state.position
      let start = this._wordStartRelativeTo(position)
      let end = position
      if(charEq(start, position)) {
        // beginning of word, move to the previous word
        let previousStart = this._wordStartRelativeTo(this._relativeChar(position, -1, 'limit'))
        // no previous word, nothing to delete
        if(start === previousStart) return
        end = this._wordEndRelativeTo(start)
      }

      // TODO delete at beginning of line deletes last word on previous line or last word of previous paragraph

      let wordChars = this.replica.getTextRange(start, end)
      this.replica.rmChars(wordChars)
      this._flow({ start: start, end: end, action: ACTION_DELETE})

      let lineContainingStart = lineContainingChar(this.state.lines, start)
      let endOfLine = lineContainingStart ? lineContainingStart.endOfLine : true
      this._setPosition(start, endOfLine)
    }
  }

  eraseWordForward() {
    if(this.state.selectionActive) {
      this._eraseSelection()
    } else {
      let position = this.state.position
      let options
      if(isWhitespace(this._relativeChar(position, 1, 'limit').char)) {
        options = { includeLeadingSpace: true }
      }
      let start = position
      let end = this._wordEndRelativeTo(start, options)
      if(charEq(end, position)) {
        // ending of word, move to the next word
        let nextEnd = this._wordEndRelativeTo(this._relativeChar(position, 1, 'limit'), options)
        // no next word, nothing to delete
        if(end === nextEnd) return
        start = this._wordStartRelativeTo(end, options)
      }

      // TODO delete at end of line deletes first word on next line or first word of next paragraph

      let wordChars = this.replica.getTextRange(start, end)
      this.replica.rmChars(wordChars)
      this._flow({ start: start, end: end, action: ACTION_DELETE})

      let lineContainingStart = lineContainingChar(this.state.lines, start)
      let endOfLine = lineContainingStart ? lineContainingStart.endOfLine : true
      this._setPosition(start, endOfLine)
    }
  }

  eraseSelection() {
    if(this.state.selectionActive) {
      this._eraseSelection()
    }
  }

  toggleBold() {
    this._toggleAttribute(ATTR.BOLD)
  }

  toggleItalics() {
    this._toggleAttribute(ATTR.ITALIC)
  }

  toggleUnderline() {
    this._toggleAttribute(ATTR.UNDERLINE)
  }

  toggleStrikethrough() {
    this._toggleAttribute(ATTR.STRIKETHROUGH)
  }

  toggleSuperscript() {
    this._toggleAttribute(ATTR.SUPERSCRIPT, ATTR.SUBSCRIPT)
  }

  toggleSubscript() {
    this._toggleAttribute(ATTR.SUBSCRIPT, ATTR.SUPERSCRIPT)
  }

  _insertChars(value, attributes, atPosition, reflow) {
    if(_.isUndefined(reflow)) reflow = true
    let position

    if(this.state.selectionActive) {
      position = this.state.selectionLeftChar
      this._eraseSelection()
    } else {
      if(atPosition) {
        position = atPosition
      } else {
        position = this.state.position
      }

      if(position === EOF) {
        position = this._relativeChar(EOF, -1, 'limit')
      }
    }

    if(!attributes) {
      if(this.state.selectionActive) {
        position = this.state.selectionLeftChar
        // if selection, then activeAttributes (set by command or toolbar) are set by the first selected char
        attributes = this._relativeChar(position, 1, 'limit').attributes
      } else {
        attributes = this.state.activeAttributes ?
          this.state.activeAttributes :
          this._relativeChar(position, 0).attributes // reload attributes from the replica in case they have changed
      }
    }

    this.replica.insertCharsAt(position, value, attributes)

    let relativeMove = value.length
    let newPosition = this._relativeChar(position, relativeMove)
    // if the last char is a newline, then we want to position on the start of the next line
    let newPositionEolStart = value.slice(-1) === '\n'

    if(reflow) this._flow({start: position, end: newPosition, action: ACTION_INSERT})

    this._setPosition(newPosition, newPositionEolStart)
    this.setState({activeAttributes: attributes})

    // return the new position so that multiple insertChars calls can be made in sequence
    return newPosition
  }

  _relativeChar(charOrId, relative, wrap) {
    return this.replica.getCharRelativeTo(charOrId, relative, wrap)
  }

  /**
   * Flows the content i.e. wraps the text into multiple lines, splits each line into chunks with the same
   * attributes, and sets the component state based on the result. This state is then used for rendering the
   * editor surface during the next render cycle. This should be called after any operation that may change
   * the content flow, such as inserting or deleting text.
   *
   * One or more spaces at the end of a word are not "counted" for wrapping purposes because that would cause the
   * space to show up at the beginning of the next line. Instead, these spaces are included on the prior line
   * even though strictly it causes the line length to exceed the margin. This behavior is consistent with
   * common word processors such as Microsoft Word and Google Docs.
   *
   * Even though all of the state here can be calculated from the replica, this is not done at render time
   * because the line state must be available when user input such as clicks or selections are made.
   *
   * Because the algorithm to shortcut the flow before and after the changed area is complex, and can fail
   * without causing an obvious problem other than performance, debug statements are included and can be
   * enabled by setting FLOW_DEBUG to true.
   */
  _flow(modification) {
    let lines = []

    let modStartAtLine
    let startChar
    if(modification) {
      let startLineIndex = 0
      modStartAtLine = lineContainingChar(this.state.lines, modification.start, true)

      // shortcut beginning of flow, start at the line prior to the one containing the modification position
      if(modStartAtLine && modStartAtLine.index > 0) {
        startLineIndex = modStartAtLine.index - 1
        // if the previous line is a hard return, we can ignore it --> no changes on this line could affect it
        if(this.state.lines[startLineIndex].isHard()) {
          startLineIndex = modStartAtLine.index
        }
        pushArray(lines, this.state.lines.slice(0, startLineIndex))
        startChar = this.state.lines[startLineIndex].start
      } else {
        startChar = BASE_CHAR
      }
    } else {
      startChar = BASE_CHAR
    }

    if(FLOW_DEBUG) {
      let modificationSummary
      if(modification) {
        modificationSummary = modification.action + ' ' + modification.start.toString() + ' … ' + modification.end.toString()
      } else {
        modificationSummary = 'n/a'
      }
      console.group(`FLOW: modification=${modificationSummary} startChar=${startChar.toString()}`)
    }

    let newLine = false

    class CurrentWord {
      constructor() {
        this.reset()
      }

      reset() {
        this.chars = []
        this.length = 0
        this.pendingChunks = []
        this.advance = 0
        this.lineAdvance = 0
        this.attributes = null
        this.lastCharSpace = false
      }

      pushChar(c, fontSize) {
        if(!this.attributes) {
          this.attributes = c.attributes
        }
        let charAdvance = TextFontMetrics.advanceXForChars(fontSize, c)

        // don't count spaces in the word in the advance, but include it in the line advance
        if(c.char === ' ') {
          this.lastCharSpace = true
        } else {
          this.advance += charAdvance
        }
        this.lineAdvance += charAdvance
        this.chars.push(c)
        this.length++
      }

      popChar() {
        this.length--
        return this.chars.pop()
      }

      pushChunks() {
        if(this.length > 0) {
          this.pendingChunks.push({
            length: this.length,
            attributes: this.attributes
          })
        }
        this.attributes = null
        this.length = 0
      }
    }

    class CurrentLine {
      constructor() {
        this.reset()
      }

      reset() {
        this.chars = []
        this.chunks = []
        this.advance = 0
        this.start = null
        this.end = null
      }

      pushWord(word) {
        pushArray(this.chars, word.chars)

        let lastChunk
        if(this.chunks.length > 0) {
          lastChunk = this.chunks[this.chunks.length - 1]
          // if the last chunk in the line matches attributes with the first word chunk, join them to minimize chunks
          if(attributesEqual(lastChunk.attributes, word.pendingChunks[0].attributes)) {
            lastChunk.end += word.pendingChunks[0].length
            word.pendingChunks.shift()
          }
        }

        // set the start and end of each chunk, start and end are suitable for feeding to chars.slice i.e. end is exclusive
        let start = lastChunk ? lastChunk.end : 0
        word.pendingChunks.map(c => {
          c.start = start
          c.end = start + c.length
          start = c.end
          delete c.length
          return c
        })

        pushArray(this.chunks, word.pendingChunks)
        this.end = this.chars[this.chars.length - 1]
        this.advance += word.lineAdvance
        word.reset()
      }

      pushNewline(c) {
        invariant(c.char === '\n', 'pushNewline can only be called with a newline char.')
        this.chars.push(c)
        this.end = c
      }

      pushEof() {
        this.end = EOF
      }
    }

    let currentWord = new CurrentWord()
    let currentLine = new CurrentLine()

    let pushLine = (line) => {
      if(line.end) {
        let l = new Line(line.chars, line.chunks, line.start, line.end, line.advance)
        lines.push(l)
        if(FLOW_DEBUG) {
          console.debug(`PUSHED LINE (index=${lines.length - 1}): ${l.toString()}`)
        }
        newLine = true
      }
      line.reset()
      line.start = lines.length > 0 ? lines[lines.length - 1].end : BASE_CHAR
    }

    let modLineOffset

    let processChar = (c) => {
      if(newLine && modification && this.state.lines) {
        // Shortcut the flow algorithm if we know and are past the line after the modifications, and
        // any existing lines past that point match up with our current buffer (word + char). Once modLineOffset
        // is defined, we'll keep doing the search for each pushed line
        if(_.isUndefined(modLineOffset)) {
          if(modification.action === ACTION_INSERT) {
            if(lines[lines.length - 1].hasChar(modification.end)) {
              // the number of lines inserted
              modLineOffset = lines.length - modStartAtLine.index
              if(modification.end.char === '\n') modLineOffset++
            }
          } else if(modification.action === ACTION_DELETE) {
            if(lines.length > modStartAtLine.index) {
              let modEndAtLine = lineContainingChar(this.state.lines, modification.end)
              modLineOffset = modStartAtLine.index - modEndAtLine.index
            }
          } else if(modification.action === ACTION_ATTRIBUTES) {
            if(lines[lines.length - 1].hasChar(modification.end)) {
              // the number of lines less or more based on the attributes change
              let modEndAtLine = lineContainingChar(this.state.lines, modification.end)
              modLineOffset = lines.length - modEndAtLine.index
            }
          }
        }
        if(!_.isUndefined(modLineOffset)) {
          let startIndex = lines.length - modLineOffset
          let matchChars = _.clone(currentWord.chars)
          matchChars.push(c)

          if(FLOW_DEBUG) {
            console.group(`MATCH ATTEMPT: chars=[${matchChars.map(ch => ch.char === '\n' ? '↵' : ch.char).join('')}] modLineOffset=${modLineOffset}`)
          }
          let iterations = 0
          let matched = false
          for(let i = startIndex; i < this.state.lines.length && !matched; i++) {
            iterations++<commit_msg>Fix nasty bug with newline insert ops failing when on the first line<commit_after>import _ from 'lodash'
import invariant from 'react/lib/invariant'

import alt from '../core/alt'
import {ATTR, attributesEqual} from '../core/attributes'

import EditorActions from './EditorActions'
import { BASE_CHAR, EOF } from '../core/RichText'
import { pushArray/*, logInGroup*/ } from '../core/utils'
import { default as tokenizer, isWhitespace } from '../core/tokenizer'
import writeHtml from '../core/htmlwriter'
import writeText from '../core/textwriter'
import TextFontMetrics from '../core/TextFontMetrics'
import { charEq, charArrayEq, charId, Line, lineContainingChar } from '../core/EditorCommon'

const FLOW_DEBUG = false
const ACTION_DELETE = 'delete'
const ACTION_INSERT = 'insert'
const ACTION_ATTRIBUTES = 'attributes'

/**
 * The EditorStore is the editor's single source of truth for application state and logic related to the editor.
 * View state updates are provided to the view components in setState callbacks. This uses the Facebook Flux
 * unidirectional flow idea: actions (from the view and elsewhere) -> dispatcher -> store -> view.
 *
 * See https://facebook.github.io/flux/docs/overview.html
 */
class EditorStore {
  constructor() {
    this.exportPublicMethods({
      getSelection: () => this.getSelection(),
      getSelectionRich: () => this.getSelectionRich(),
      getSelectionHtml: () => this.getSelectionHtml(),
      getSelectionText: () => this.getSelectionText()
    })

    this.bindActions(EditorActions)

    this.config = null
    this.replica = null

    this.state = {
      position: BASE_CHAR,
      positionEolStart: true,
      cursorMotion: false,
      selectionActive: false,
      remoteCursors: {},
      remoteNameReveal: [],
      focus: false,
      loaded: false,
      lines: []
    }
  }

  initialize({config, replica}) {
    TextFontMetrics.setConfig(config)
    this.config = config
    this.replica = replica

    this.setState({focus: config.initialFocus})
  }

  initializeCursorModel({cursorSet, cursorModel}) {
    this.cursorSet = cursorSet
    this.cursorModel = cursorModel
    this._setRemoteCursorModel()
  }

  replicaInitialized() {
    this.setState({loaded: true})
    this._flow()
  }

  replicaUpdated() {
    // TODO can we determine what changed and do a more focused flow via the modifications parameter?
    this._flow()
  }

  focusInput() {
    this._delayedCursorBlink(0)
    this.setState({focus: true})
  }

  inputFocusLost() {
    this.setState({focus: false})
  }

  setRemoteCursorPosition(remoteCursor) {
    let id = remoteCursor.model._id
    // some bug in Swarm.js: sometimes the id is blank if a lot of cursor events have happened quickly in succession
    if(!id) return
    let remoteCursors = this.state.remoteCursors
    remoteCursors[id] = remoteCursor
    this.setState({remoteCursors: remoteCursors})
    this._delayedRemoteCursorNameReveal(id)
  }

  unsetRemoteCursorPosition(remoteCursor) {
    let id = remoteCursor.model._id
    // some bug in Swarm.js: sometimes the id is blank if a lot of cursor events have happened quickly in succession
    if(!id) return
    let remoteCursors = this.state.remoteCursors
    if(id in remoteCursors) {
      delete remoteCursors[id]
      this.setState({remoteCursors: remoteCursors})
    }
  }

  updateRemoteCursorModel() {
    this._setRemoteCursorModel()
  }

  revealRemoteCursorName(remoteCursor) {
    this._delayedRemoteCursorNameReveal(remoteCursor.model._id)
  }

  navigateLeft() {
    this._navigateLeftRight(-1)
  }

  navigateRight() {
    this._navigateLeftRight(1)
  }

  navigateUp() {
    this._navigateUpDown(-1)
  }

  navigateDown() {
    this._navigateUpDown(1)
  }

  navigatePageUp() {
    // assume for now a page is 10 lines
    this._navigateUpDown(-10)
  }

  navigatePageDown() {
    // assume for now a page is 10 lines
    this._navigateUpDown(10)
  }

  navigateStart() {
    this._resetPosition()
  }

  navigateStartLine() {
    if(this._emptyEditor()) {
      this._resetPosition()
      return
    }

    let {line} = lineContainingChar(this.state.lines, this.state.position, this.state.positionEolStart)
    this._setPosition(line.start, true)
  }

  navigateEnd() {
    if(this._emptyEditor()) {
      this._resetPosition()
      return
    }

    let positionEolStart = false
    if(this._lastLine().isEof()) {
      positionEolStart = true
    }
    this._setPosition(this._relativeChar(BASE_CHAR, -1), positionEolStart)
  }

  navigateEndLine() {
    if(this._emptyEditor()) {
      this._resetPosition()
      return
    }

    let {line} = lineContainingChar(this.state.lines, this.state.position, this.state.positionEolStart)
    let position
    let positionEolStart = false
    if(line.isEof() || line.chunks.length === 0) {
      position = this.state.position
      positionEolStart = true
    } else if (line.isHard()) {
      position = this._relativeChar(line.end, -1, 'limit')
    } else {
      position = line.end
    }
    this._setPosition(position, positionEolStart)
  }

  navigateWordLeft() {
    this._navigateWordLeftRight(-1)
  }

  navigateWordRight() {
    this._navigateWordLeftRight(1)
  }

  navigateToCoordinates(coordinates) {
    let {position, positionEolStart} = this._coordinatesToPosition(coordinates)

    // set the position and selection anchor if the user continues the selection later
    position = position ? position : BASE_CHAR

    this._setPosition(position, positionEolStart)
    this.setState({
      selectionAnchorChar: position
    })
  }

  selectionLeft() {
    this._selectionLeftRight(-1)
  }

  selectionRight() {
    this._selectionLeftRight(1)
  }

  selectionUp() {
    this._selectionUpDown(-1)
  }

  selectionDown() {
    this._selectionUpDown(1)
  }

  selectionPageUp() {
    // assume for now a page is 10 lines
    this._selectionUpDown(-10)
  }

  selectionPageDown() {
    // assume for now a page is 10 lines
    this._selectionUpDown(10)
  }

  selectionStart() {
    this._modifySelection(BASE_CHAR, true)
  }

  selectionStartLine() {
    let {line} = lineContainingChar(this.state.lines, this.state.position, this.state.positionEolStart)
    this._modifySelection(line.start, true)
  }

  selectionEnd() {
    let toChar = this._lastLine() && this._lastLine().isEof() ? EOF : this._relativeChar(BASE_CHAR, -1)
    this._modifySelection(toChar, toChar === EOF)
  }

  /**
   * Google docs behavior (as of 2015-06-22) is:
   * - line with soft return, select line with space at end
   * - line with hard return, select line without selecting the hard return (no action on empty lines)
   * - EOF line, no action
   *
   * Word 2010 behavior is:
   * - line with soft return, select line with space at end
   * - line with hard return, select line including the hard return (same on empty lines)
   * - EOF line, show "space" selection at EOF containing a newline
   *
   * We implement the Google docs behavior here, which seems a bit more intuitive.
   */
  selectionEndLine() {
    let {line} = lineContainingChar(this.state.lines, this.state.position, this.state.positionEolStart)
    let toChar
    let positionEolStart = false
    if(line.isEof()) {
      toChar = this.state.position
      positionEolStart = true
    } else if (line.isHard()) {
      toChar = this._relativeChar(line.end, -1, 'limit')
    } else {
      toChar = line.end
    }
    this._modifySelection(toChar, positionEolStart)
  }

  selectionWordLeft() {
    let position
    let positionEolStart
    if(this.state.position === EOF) {
      position = this._lastLine().start
      positionEolStart = true
    } else {
      position = this._wordStartRelativeTo(this.state.position)
      positionEolStart = !lineContainingChar(this.state.lines, this.state.position).endOfLine
    }
    this._modifySelection(position, positionEolStart)
  }

  selectionWordRight() {
    let position = this._wordEndRelativeTo(this.state.position)
    let endOfLine = lineContainingChar(this.state.lines, this.state.position).endOfLine

    this._modifySelection(position, !endOfLine)
  }

  selectionAll() {
    this._setPosition(BASE_CHAR)
    if(this.state.lines.length === 0) {
      this._modifySelection(EOF, true)
    } else {
      let lastChar = this._lastLine() && this._lastLine().isEof() ? EOF : this._lastLine().end
      this._modifySelection(lastChar, false)
    }
  }

  selectToCoordinates(coordinates) {
    let {position, positionEolStart} = this._coordinatesToPosition(coordinates)
    this._modifySelection(position, positionEolStart)
  }

  /**
   * Word selection follows Google Docs and Microsoft Word 2010 behavior of:
   * - selecting the word that was clicked, and one or more spaces following it
   * - if clicking at the end of a line with a soft return, the first word on the next line is selected
   * - if clicking at the end of a line with a hard return, the hard return is selected (shown as a "space")
   * - if clicking at the end of the last (empty) line, a (non-existent) hard return at EOF is selected (shown as
   *   a "space")
   */
  selectWordAtCurrentPosition() {
    if(this.state.lines.length === 0
      || lineContainingChar(this.state.lines, this.state.position, true).line.isEof()) {
      this._setPosition(this.state.position)
      this._modifySelection(EOF, true)
    } else {
      let word = this._wordRelativeTo(this.state.position)
      this._setPosition(word.start)
      this._modifySelection(word.end, false)
    }
  }

  copySelection(copyHandler) {
    let selectionChunks = this._getSelectionRich()
    if(selectionChunks && selectionChunks.length > 0) {
      let copiedText = writeText(selectionChunks)
      let copiedHtml = writeHtml(selectionChunks)
      copyHandler(copiedText, copiedHtml, selectionChunks)
    }
  }

  getSelection() {
    return this.replica.getTextRange(this.state.selectionLeftChar, this.state.selectionRightChar)
  }

  getSelectionRich() {
    return this._getSelectionRich()
  }

  getSelectionHtml() {
    return writeHtml(this._getSelectionRich())
  }

  getSelectionText() {
    return writeText(this._getSelectionRich())
  }

  insertChars({value, attributes, atPosition, reflow}) {
    return this._insertChars(value, attributes, atPosition, reflow)
  }

  insertCharsBatch(chunks) {
    let firstInsertPosition = this.state.position
    let insertPosition = null
    chunks.forEach(c => {
      insertPosition = this._insertChars(c.text, c.attrs || {}, insertPosition, false)
    })
    this._flow({start: firstInsertPosition, end: insertPosition, action: 'insert'})
  }

  eraseCharBack() {
    if(this.state.selectionActive) {
      this._eraseSelection()
    } else if(!charEq(this.state.position, BASE_CHAR)) {
      let position = this._relativeChar(this.state.position, -1)
      this.replica.rmChars(this.state.position)
      this._flow({ start: position, end: this.state.position, action: ACTION_DELETE})

      let endOfLine = lineContainingChar(this.state.lines, position).endOfLine
      this._setPosition(position, endOfLine)
    }
  }

  eraseCharForward() {
    if(this.state.selectionActive) {
      this._eraseSelection()
    } else if(!this._cursorAtEnd()) {
      let next = this._relativeChar(this.state.position, 1, 'limit')
      this.replica.rmChars(next)
      this._flow({ start: this.state.position, end: next, action: ACTION_DELETE})

      let endOfLine = lineContainingChar(this.state.lines, this.state.position).endOfLine
      this._setPosition(this.state.position, endOfLine)
    }
  }

  eraseWordBack() {
    if(this.state.selectionActive) {
      this._eraseSelection()
    } else {
      let position = this.state.position
      let start = this._wordStartRelativeTo(position)
      let end = position
      if(charEq(start, position)) {
        // beginning of word, move to the previous word
        let previousStart = this._wordStartRelativeTo(this._relativeChar(position, -1, 'limit'))
        // no previous word, nothing to delete
        if(start === previousStart) return
        end = this._wordEndRelativeTo(start)
      }

      // TODO delete at beginning of line deletes last word on previous line or last word of previous paragraph

      let wordChars = this.replica.getTextRange(start, end)
      this.replica.rmChars(wordChars)
      this._flow({ start: start, end: end, action: ACTION_DELETE})

      let lineContainingStart = lineContainingChar(this.state.lines, start)
      let endOfLine = lineContainingStart ? lineContainingStart.endOfLine : true
      this._setPosition(start, endOfLine)
    }
  }

  eraseWordForward() {
    if(this.state.selectionActive) {
      this._eraseSelection()
    } else {
      let position = this.state.position
      let options
      if(isWhitespace(this._relativeChar(position, 1, 'limit').char)) {
        options = { includeLeadingSpace: true }
      }
      let start = position
      let end = this._wordEndRelativeTo(start, options)
      if(charEq(end, position)) {
        // ending of word, move to the next word
        let nextEnd = this._wordEndRelativeTo(this._relativeChar(position, 1, 'limit'), options)
        // no next word, nothing to delete
        if(end === nextEnd) return
        start = this._wordStartRelativeTo(end, options)
      }

      // TODO delete at end of line deletes first word on next line or first word of next paragraph

      let wordChars = this.replica.getTextRange(start, end)
      this.replica.rmChars(wordChars)
      this._flow({ start: start, end: end, action: ACTION_DELETE})

      let lineContainingStart = lineContainingChar(this.state.lines, start)
      let endOfLine = lineContainingStart ? lineContainingStart.endOfLine : true
      this._setPosition(start, endOfLine)
    }
  }

  eraseSelection() {
    if(this.state.selectionActive) {
      this._eraseSelection()
    }
  }

  toggleBold() {
    this._toggleAttribute(ATTR.BOLD)
  }

  toggleItalics() {
    this._toggleAttribute(ATTR.ITALIC)
  }

  toggleUnderline() {
    this._toggleAttribute(ATTR.UNDERLINE)
  }

  toggleStrikethrough() {
    this._toggleAttribute(ATTR.STRIKETHROUGH)
  }

  toggleSuperscript() {
    this._toggleAttribute(ATTR.SUPERSCRIPT, ATTR.SUBSCRIPT)
  }

  toggleSubscript() {
    this._toggleAttribute(ATTR.SUBSCRIPT, ATTR.SUPERSCRIPT)
  }

  _insertChars(value, attributes, atPosition, reflow) {
    if(_.isUndefined(reflow)) reflow = true
    let position

    if(this.state.selectionActive) {
      position = this.state.selectionLeftChar
      this._eraseSelection()
    } else {
      if(atPosition) {
        position = atPosition
      } else {
        position = this.state.position
      }

      if(position === EOF) {
        position = this._relativeChar(EOF, -1, 'limit')
      }
    }

    if(!attributes) {
      if(this.state.selectionActive) {
        position = this.state.selectionLeftChar
        // if selection, then activeAttributes (set by command or toolbar) are set by the first selected char
        attributes = this._relativeChar(position, 1, 'limit').attributes
      } else {
        attributes = this.state.activeAttributes ?
          this.state.activeAttributes :
          this._relativeChar(position, 0).attributes // reload attributes from the replica in case they have changed
      }
    }

    this.replica.insertCharsAt(position, value, attributes)

    let relativeMove = value.length
    let newPosition = this._relativeChar(position, relativeMove)
    // if the last char is a newline, then we want to position on the start of the next line
    let newPositionEolStart = value.slice(-1) === '\n'

    if(reflow) this._flow({start: position, end: newPosition, action: ACTION_INSERT})

    this._setPosition(newPosition, newPositionEolStart)
    this.setState({activeAttributes: attributes})

    // return the new position so that multiple insertChars calls can be made in sequence
    return newPosition
  }

  _relativeChar(charOrId, relative, wrap) {
    return this.replica.getCharRelativeTo(charOrId, relative, wrap)
  }

  /**
   * Flows the content i.e. wraps the text into multiple lines, splits each line into chunks with the same
   * attributes, and sets the component state based on the result. This state is then used for rendering the
   * editor surface during the next render cycle. This should be called after any operation that may change
   * the content flow, such as inserting or deleting text.
   *
   * One or more spaces at the end of a word are not "counted" for wrapping purposes because that would cause the
   * space to show up at the beginning of the next line. Instead, these spaces are included on the prior line
   * even though strictly it causes the line length to exceed the margin. This behavior is consistent with
   * common word processors such as Microsoft Word and Google Docs.
   *
   * Even though all of the state here can be calculated from the replica, this is not done at render time
   * because the line state must be available when user input such as clicks or selections are made.
   *
   * Because the algorithm to shortcut the flow before and after the changed area is complex, and can fail
   * without causing an obvious problem other than performance, debug statements are included and can be
   * enabled by setting FLOW_DEBUG to true.
   */
  _flow(modification) {
    let lines = []

    let modStartAtLine
    let startChar
    if(modification) {
      let startLineIndex = 0
      modStartAtLine = lineContainingChar(this.state.lines, modification.start, true)

      // shortcut beginning of flow, start at the line prior to the one containing the modification position
      if(modStartAtLine && modStartAtLine.index > 0) {
        startLineIndex = modStartAtLine.index - 1
        // if the previous line is a hard return, we can ignore it --> no changes on this line could affect it
        if(this.state.lines[startLineIndex].isHard()) {
          startLineIndex = modStartAtLine.index
        }
        pushArray(lines, this.state.lines.slice(0, startLineIndex))
        startChar = this.state.lines[startLineIndex].start
      } else {
        startChar = BASE_CHAR
      }
    } else {
      startChar = BASE_CHAR
    }

    if(FLOW_DEBUG) {
      let modificationSummary
      if(modification) {
        modificationSummary = modification.action + ' ' + modification.start.toString() + ' … ' + modification.end.toString()
      } else {
        modificationSummary = 'n/a'
      }
      console.group(`FLOW: modification=${modificationSummary} startChar=${startChar.toString()}`)
    }

    let newLine = false

    class CurrentWord {
      constructor() {
        this.reset()
      }

      reset() {
        this.chars = []
        this.length = 0
        this.pendingChunks = []
        this.advance = 0
        this.lineAdvance = 0
        this.attributes = null
        this.lastCharSpace = false
      }

      pushChar(c, fontSize) {
        if(!this.attributes) {
          this.attributes = c.attributes
        }
        let charAdvance = TextFontMetrics.advanceXForChars(fontSize, c)

        // don't count spaces in the word in the advance, but include it in the line advance
        if(c.char === ' ') {
          this.lastCharSpace = true
        } else {
          this.advance += charAdvance
        }
        this.lineAdvance += charAdvance
        this.chars.push(c)
        this.length++
      }

      popChar() {
        this.length--
        return this.chars.pop()
      }

      pushChunks() {
        if(this.length > 0) {
          this.pendingChunks.push({
            length: this.length,
            attributes: this.attributes
          })
        }
        this.attributes = null
        this.length = 0
      }
    }

    class CurrentLine {
      constructor() {
        this.reset()
      }

      reset() {
        this.chars = []
        this.chunks = []
        this.advance = 0
        this.start = null
        this.end = null
      }

      pushWord(word) {
        pushArray(this.chars, word.chars)

        let lastChunk
        if(this.chunks.length > 0) {
          lastChunk = this.chunks[this.chunks.length - 1]
          // if the last chunk in the line matches attributes with the first word chunk, join them to minimize chunks
          if(attributesEqual(lastChunk.attributes, word.pendingChunks[0].attributes)) {
            lastChunk.end += word.pendingChunks[0].length
            word.pendingChunks.shift()
          }
        }

        // set the start and end of each chunk, start and end are suitable for feeding to chars.slice i.e. end is exclusive
        let start = lastChunk ? lastChunk.end : 0
        word.pendingChunks.map(c => {
          c.start = start
          c.end = start + c.length
          start = c.end
          delete c.length
          return c
        })

        pushArray(this.chunks, word.pendingChunks)
        this.end = this.chars[this.chars.length - 1]
        this.advance += word.lineAdvance
        word.reset()
      }

      pushNewline(c) {
        invariant(c.char === '\n', 'pushNewline can only be called with a newline char.')
        this.chars.push(c)
        this.end = c
      }

      pushEof() {
        this.end = EOF
      }
    }

    let currentWord = new CurrentWord()
    let currentLine = new CurrentLine()

    let pushLine = (line) => {
      if(line.end) {
        let l = new Line(line.chars, line.chunks, line.start, line.end, line.advance)
        lines.push(l)
        if(FLOW_DEBUG) {
          console.debug(`PUSHED LINE (index=${lines.length - 1}): ${l.toString()}`)
        }
        newLine = true
      }
      line.reset()
      line.start = lines.length > 0 ? lines[lines.length - 1].end : BASE_CHAR
    }

    let modLineOffset

    let processChar = (c) => {
      if(newLine && modification && this.state.lines) {
        // Shortcut the flow algorithm if we know and are past the line after the modifications, and
        // any existing lines past that point match up with our current buffer (word + char). Once modLineOffset
        // is defined, we'll keep doing the search for each pushed line
        if(_.isUndefined(modLineOffset)) {
          if(modification.action === ACTION_INSERT) {
            if(lines[lines.length - 1].hasChar(modification.end)) {
              // the number of lines inserted
              modLineOffset = lines.length - modStartAtLine.index
              if(modification.end.char === '\n') modLineOffset++
            }
          } else if(modification.action === ACTION_DELETE) {
            if(lines.length > modStartAtLine.index) {
              let modEndAtLine = lineContainingChar(this.state.lines, modification.end)
              modLineOffset = modStartAtLine.index - modEndAtLine.index
            }
          } else if(modification.action === ACTION_ATTRIBUTES) {
            if(lines[lines.length - 1].hasChar(modification.end)) {
              // the number of lines less or more based on the attributes change
              let modEndAtLine = lineContainingChar(this.state.lines, modification.end)
              modLineOffset = lines.length - modEndAtLine.index
            }
          }
        }
        if(!_.isUndefined(modLineOffset)) {
          let startIndex = modLineOffset > lines.length ? 0 : lines.length - modLineOffset
          let matchChars = _.clone(currentWord.chars)
          matchChars.push(c)

          if(FLOW_DEBUG) {
            console.group(`MATCH ATTEMPT: chars=[${matchChars.map(ch => ch.char === '\n' ? '↵' : ch.char).join('')}] modLineOffset=${modLineOffset}`)
          }
          let iterations = 0
          let matched = false
          for(let i = startIndex; i < this.state.lines.length && !matched; i++) {
            iterations++
{'fraction_non_alphanumeric': 0.10174504584442473, 'fraction_numerical': 0.00348586639624794, 'mean_word_length': 2.7513869075923285, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 42, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7213574', 'n_tokens_mistral': 14029, 'n_tokens_neox': 13669, 'n_words': 3574}