<commit_before>import {Node, Pos, style} from "../src/model"

let inlineContext = null

function buildInline(style) {
  return function() {
    return {type: "inline", style: style, content: arguments}
  }
}

function build(type, attrs) {
  return function() {
    return {type: "block", style: type, content: arguments, attrs: attrs}
  }
}

let styles = []

function pushInto(node, values) {
  if (Array.isArray(values))
    for (var i = 0; i < values.length; i++)
      node.push(values[i])
  else
    node.push(values)
}

function parseDoc(value) {
  if (typeof value == "string") {
    let re = /<(\w+)>/g, m, out = "", pos = 0
    let node = new Node.Inline("text", styles, "")
    while (m = re.exec(value)) {
      node.text += value.slice(pos, m.index)
      pos = m.index + m[0].length
      tags[m[1]] = {node: node, offset: node.text.length}
    }
    node.text += value.slice(pos)
    return node
  } else if (value.type == "inline") {
    let start = styles, result = []
    styles = style.add(styles, value.style)
    for (let i = 0; i < value.content.length; i++)
      result = result.concat(parseDoc(value.content[i]))
    styles = start
    return result
  } else if (value.type == "insert") {
    let type = Node.types[value.style]
    if (type.type == "inline")
      return new Node.Inline(type, styles, value.text, value.attrs)
    else
      return new Node(type, value.content, value.attrs)
  } else {
    let node = new Node(value.style, null, value.attrs)
    styles = []
    for (let i = 0; i < value.content.length; i++)
      pushInto(node, parseDoc(value.content[i]))
    return node
  }
}

let tags = Object.create(null)

function locateTags(doc, tags) {
  let path = [], offset = 0, result = {}
  function scan(node) {
    for (let tag in tags)
      if (tags[tag].node == node)
        result[tag] = new Pos(path.slice(), tags[tag].offset + offset)
    if (node.type.type == "block") offset = 0
    for (let i = 0; i < node.content.length; i++) {
      let child = node.content[i]
      if (node.type.contains != "inline") path.push(i)
      scan(child)
      if (node.type.contains != "inline") path.pop()
      else offset += child.size
      if (child.type == Node.types.text && child.text == "")
        node.content.splice(i--, 1)
    }
  }
  scan(doc)
  return result
}

export function doc() {
  let doc = parseDoc(build("doc").apply(null, arguments))
  doc.tag = locateTags(doc, tags)
  tags = Object.create(null)
  return doc
}
<commit_msg>Overhaul test/build.js to allow tags at arbitrary positions<commit_after>import {Node, Pos, style} from "../src/model"

let inlineContext = null

function buildInline(style) {
  return function() {
    return {type: "inline", style: style, content: arguments}
  }
}

function build(type, attrs) {
  return function() {
    return {type: "block", style: type, content: arguments, attrs: attrs}
  }
}

let styles = []

function parseDoc(value, target, path) {
  if (typeof value == "string") {
    let offset = target.maxOffset
    let re = /<(\w+)>/g, m, pos = 0, out = ""
    while (m = re.exec(value)) {
      out += value.slice(pos, m.index)
      pos = m.index + m[0].length
      tags[m[1]] = new Pos(path, offset + out.length)
    }
    out += value.slice(pos)
    if (out) target.push(Node.text(out, styles))
  } else if (value.type == "inline") {
    let start = styles, result = []
    styles = style.add(styles, value.style)
    for (let i = 0; i < value.content.length; i++)
      parseDoc(value.content[i], target, path)
    styles = start
  } else if (value.type == "insert") {
    let type = Node.types[value.style]
    if (type.type == "inline")
      target.push(new Node.Inline(type, styles, value.text, value.attrs))
    else
      target.push(new Node(type, value.content, value.attrs))
  } else {
    let node = new Node(value.style, null, value.attrs)
    let nodePath = path.concat(target.maxOffset)
    styles = []
    for (let i = 0; i < value.content.length; i++)
      parseDoc(value.content[i], node, nodePath)
    target.push(node)
  }
}

let tags = Object.create(null)

export function doc() {
  let doc = new Node("doc")
  for (let i = 0; i < arguments.length; i++)
    parseDoc(arguments[i], doc, [])
  doc.tag = tags
  tags = Object.create(null)
  return doc
}

{'fraction_non_alphanumeric': 0.16190921853065043, 'fraction_numerical': 0.003743565746373421, 'mean_word_length': 2.925619834710744, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2678150', 'n_tokens_mistral': 1456, 'n_tokens_neox': 1452, 'n_words': 399}