<commit_before>#= require underscore
#= require linked_list
#= require tandem/line


class TandemDocument
  @INDENT_PREFIX: 'indent-'


  @normalizeHtml: (root, options = {}) ->
    if root.childNodes.length == 0
      div = root.ownerDocument.createElement('div')
      return root.appendChild(div)
    _.each(_.clone(root.childNodes), (child) =>
      if child.nodeType != child.ELEMENT_NODE
        child.parentNode.removeChild(child)
      else if options.ignoreDirty || child.classList.contains(Tandem.Line.DIRTY_CLASS) || true
        # TODO editor.update should mark dirty lines
        Tandem.Line.normalizeHtml(child)
    )


  constructor: (@editor, @root) ->
    this.buildLines()

  appendLine: (lineNode) ->
    return this.insertLineBefore(lineNode, null)

  applyAttribute: (index, length, attr, value) ->
    [startLine, startLineOffset] = this.findLineAtOffset(index)
    [endLine, endLineOffset] = this.findLineAtOffset(index + length)
    if startLine == endLine
      startLine.applyAttribute(startLineOffset, endLineOffset, attr, value)
    else
      curLine = startLine.next
      while curLine? && curLine != endLine
        next = curLine.next
        curLine.applyAttribute(0, curLine.length, attr, value)
        curLine = next
      startLine.applyAttribute(startLineOffset, startLine.length, attr, value)
      endLine.applyAttribute(0, endLineOffset, attr, value) if endLine?

  buildLines: ->
    this.reset()
    TandemDocument.normalizeHtml(@root, {ignoreDirty: true})
    this.rebuild()

  cleanNode: (lineNode) ->
    line = this.findLine(lineNode)
    if line? && this.updateLine(line)
      lineNode.classList.remove(Tandem.Line.DIRTY_CLASS)
      return true
    return false

  deleteText: (index, length) ->
    if index + length == @length
      @trailingNewline = false
      @length -= 1
      length -= 1
    return if length <= 0
    [startLineNode, startOffset] = Tandem.Utils.getChildAtOffset(@root, index)
    [endLineNode, endOffset] = Tandem.Utils.getChildAtOffset(@root, index + length)
    fragment = Tandem.Utils.extractNodes(startLineNode, startOffset, endLineNode, endOffset)
    lineNodes = _.values(fragment.childNodes).concat(_.uniq([startLineNode, endLineNode]))
    _.each(lineNodes, (lineNode) =>
      line = this.findLine(lineNode)
      this.updateLine(line) if line?
    )

  detectChange: ->
    # Go through HTML (which should be normalized)
    # Make sure non are different from their innerHTML, if so record change
    # Returns changes made

  findLeaf: (node) ->
    lineNode = node.parentNode
    while lineNode? && !Tandem.Line.isLineNode(lineNode)
      lineNode = lineNode.parentNode
    return null if !lineNode?
    line = this.findLine(lineNode)
    return line.findLeaf(node)

  findLine: (node) ->
    node = this.findLineNode(node)
    return @lineMap[node.id]

  findLineAtOffset: (offset) ->
    retLine = @lines.first
    _.all(@lines.toArray(), (line, index) =>
      retLine = line
      if offset < line.length + 1
        return false
      else
        offset -= (line.length + 1) if index < @lines.length - 1
        return true
    )
    return [retLine, offset]

  findLineNode: (node) ->
    while node? && !Tandem.Line.isLineNode(node)
      node = node.parentNode
    return node

  forceTrailingNewline: ->
    unless @trailingNewline
      if @lines.length > 1 && @lines.last.length == 0
        @root.removeChild(@lines.last.node)
        this.removeLine(@lines.last)
      @trailingNewline = true
      @length += 1

  insertLineBefore: (newLineNode, refLine) ->
    line = new Tandem.Line(this, newLineNode)
    if refLine != null
      @lines.insertAfter(refLine.prev, line)
    else
      @lines.append(line)
    @lineMap[line.id] = line
    @length += line.length
    @length += 1 if @lines.length > 1
    return line

  insertText: (index, text) ->
    [line, lineOffset] = this.findLineAtOffset(index)
    textLines = text.split("\n")
    if index == @length
      @trailingNewline = false
      @length -= 1    # Doc did not get shorter but _.each loop compensates
      _.each(textLines, (textLine) =>
        contents = this.makeLineContents(textLine)
        div = @root.ownerDocument.createElement('div')
        _.each(contents, (content) ->
          div.appendChild(content)
        )
        @root.appendChild(div)
        this.appendLine(div)<commit_msg>fuzzer passing, save for occasional not enough marker errors<commit_after>#= require underscore
#= require linked_list
#= require tandem/line


class TandemDocument
  @INDENT_PREFIX: 'indent-'


  @normalizeHtml: (root, options = {}) ->
    if root.childNodes.length == 0
      div = root.ownerDocument.createElement('div')
      return root.appendChild(div)
    _.each(_.clone(root.childNodes), (child) =>
      if child.nodeType != child.ELEMENT_NODE
        child.parentNode.removeChild(child)
      else if options.ignoreDirty || child.classList.contains(Tandem.Line.DIRTY_CLASS) || true
        # TODO editor.update should mark dirty lines
        Tandem.Line.normalizeHtml(child)
    )


  constructor: (@editor, @root) ->
    this.buildLines()

  appendLine: (lineNode) ->
    return this.insertLineBefore(lineNode, null)

  applyAttribute: (index, length, attr, value) ->
    [startLine, startLineOffset] = this.findLineAtOffset(index)
    [endLine, endLineOffset] = this.findLineAtOffset(index + length)
    if startLine == endLine
      startLine.applyAttribute(startLineOffset, endLineOffset, attr, value)
    else
      curLine = startLine.next
      while curLine? && curLine != endLine
        next = curLine.next
        curLine.applyAttribute(0, curLine.length, attr, value)
        curLine = next
      startLine.applyAttribute(startLineOffset, startLine.length, attr, value)
      endLine.applyAttribute(0, endLineOffset, attr, value) if endLine?

  buildLines: ->
    this.reset()
    TandemDocument.normalizeHtml(@root, {ignoreDirty: true})
    this.rebuild()

  cleanNode: (lineNode) ->
    line = this.findLine(lineNode)
    if line? && this.updateLine(line)
      lineNode.classList.remove(Tandem.Line.DIRTY_CLASS)
      return true
    return false

  deleteText: (index, length) ->
    if index + length == @length
      @trailingNewline = false
      @length -= 1
      length -= 1
    return if length <= 0
    [startLineNode, startOffset] = Tandem.Utils.getChildAtOffset(@root, index)
    [endLineNode, endOffset] = Tandem.Utils.getChildAtOffset(@root, index + length)
    fragment = Tandem.Utils.extractNodes(startLineNode, startOffset, endLineNode, endOffset)
    lineNodes = _.values(fragment.childNodes).concat(_.uniq([startLineNode, endLineNode]))
    _.each(lineNodes, (lineNode) =>
      line = this.findLine(lineNode)
      this.updateLine(line) if line?
    )

  detectChange: ->
    # Go through HTML (which should be normalized)
    # Make sure non are different from their innerHTML, if so record change
    # Returns changes made

  findLeaf: (node) ->
    lineNode = node.parentNode
    while lineNode? && !Tandem.Line.isLineNode(lineNode)
      lineNode = lineNode.parentNode
    return null if !lineNode?
    line = this.findLine(lineNode)
    return line.findLeaf(node)

  findLine: (node) ->
    node = this.findLineNode(node)
    return @lineMap[node.id]

  findLineAtOffset: (offset) ->
    retLine = @lines.first
    _.all(@lines.toArray(), (line, index) =>
      retLine = line
      if offset < line.length + 1
        return false
      else
        offset -= (line.length + 1) if index < @lines.length - 1
        return true
    )
    return [retLine, offset]

  findLineNode: (node) ->
    while node? && !Tandem.Line.isLineNode(node)
      node = node.parentNode
    return node

  forceTrailingNewline: ->
    unless @trailingNewline
      if @lines.length > 1 && @lines.last.length == 0
        @root.removeChild(@lines.last.node)
        this.removeLine(@lines.last)
      @trailingNewline = true
      @length += 1

  insertLineBefore: (newLineNode, refLine) ->
    line = new Tandem.Line(this, newLineNode)
    if refLine != null
      @lines.insertAfter(refLine.prev, line)
    else
      @lines.append(line)
    @lineMap[line.id] = line
    @length += line.length
    @length += 1 if @lines.length > 1
    return line

  insertText: (index, text) ->
    [line, lineOffset] = this.findLineAtOffset(index)
    textLines = text.split("\n")
    if index == @length && @trailingNewline
      @trailingNewline = false
      @length -= 1    # Doc did not get shorter but _.each loop compensates
      _.each(textLines, (textLine) =>
        contents = this.makeLineContents(textLine)
        div = @root.ownerDocument.createElement('div')
        _.each(contents, (content) ->
          div.appendChild(content)
        )
        @root.appendChild(div)
        this.appendLine(div)
{'fraction_non_alphanumeric': 0.11649824003633473, 'fraction_numerical': 0.003406381287612127, 'mean_word_length': 2.9981842941443486, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7088067', 'n_tokens_mistral': 2756, 'n_tokens_neox': 2680, 'n_words': 624}