<commit_before>module.exports = split
import visit from 'unist-util-visit'

const ROOT_HEADING_DEPTH = 1
function split (tree, {splitDepth = ROOT_HEADING_DEPTH,
  introductionAsProperty = true, conclusionAsProperty = false}) {
  const firstHeading = find(tree, {type: 'heading', depth: ROOT_HEADING_DEPTH})
  if (!firstHeading) {
    throw new Error('No heading')
  }
  return splitInDepth(tree, ROOT_HEADING_DEPTH, splitDepth, introductionAsProperty,
    conclusionAsProperty)
}

function splitInDepth (tree, depth, splitDepth = ROOT_HEADING_DEPTH,
  introductionAsProperty = true, conclusionAsProperty = false) {
  const splitter = new Splitter()
  splitter.depth = depth
  const hasHeading = !!find(tree, {type: 'heading', depth: depth})
  if (!hasHeading) {
    return tree
  }
  visit(tree, null, (node, index, parent) => splitter.visit(node, index, parent))
  if (conclusionAsProperty) {
    splitter.extractConclusions()
  }
  if (introductionAsProperty) {
    splitter.extractIntroductions()
  }
  if (depth < splitDepth) {
    for (let i = 0; i < splitter.subTrees.length; i++) {
      const newTree = {
        type: 'root',
        children: splitter.subTrees[i].children.children,
      }
      const result = splitInDepth(newTree, depth + 1, splitDepth,
        introductionAsProperty, conclusionAsProperty)
      splitter.subTrees[i].children = result
    }
  }
  return {
    introduction: splitter.introduction,
    trees: splitter.subTrees,
  }
}

function find (tree, {type = 'heading', depth = ROOT_HEADING_DEPTH}) {
  for (let i = 0; i < tree.children.length; ++i) {
    if (tree.children[i].type === type && tree.children[i].depth === depth) {
      return tree.children[i]
    }
  }
  return null
}

class Splitter {
  constructor (depth = 1) {
    this.lastIndex = -1
    this.subTrees = []
    this.depth = depth
    this._introduction = {
      type: 'root',
      children: [],
    }
  }
  set introduction (nodes) {
    this._introduction = {type: 'root', children: nodes}
  }
  get introduction () {
    return this._introduction
  }
  extractConclusions () {
    this.subTrees.forEach(splittedPart => {
      const firstHeading = find(splittedPart.children, {
        type: 'heading',
        depth: this.depth + 1,
      })
      let lastHeading = null
      let lastIndex = 0
      for (let i = splittedPart.children.length - 1; i > 0; --i) {
        if (splittedPart.children[i] === 'heading' &&
          splittedPart.children[i].depth === this.depth - 1) {
          lastHeading = splittedPart.children[i]
          lastIndex = i
          break
        }
      }
      if (!lastHeading || lastHeading === firstHeading) {
        return
      }
      splittedPart.conclusion = {
        type: 'root',
        children: splittedPart.children.splice(lastIndex, splittedPart.children.length - lastIndex),
      }
    })
  }
  extractIntroductions () {
    const self = this
    this.subTrees.forEach(
      (subTree) => {
        const firstHeading = find(subTree.children, {
          type: 'heading',
          depth: self.depth + 1,
        })
        if (firstHeading) {
          subTree.introduction = {
            type: 'root',
            children: subTree.children.children.splice(0,
              subTree.children.children.indexOf(firstHeading)),
          }
        }
      }
    )
  }
<commit_msg>mdast-util-split: code review<commit_after>import visit from 'unist-util-visit'

const ROOT_HEADING_DEPTH = 1

module.exports = function split (tree, config) {
  const {
    splitDepth = ROOT_HEADING_DEPTH,
    introductionAsProperty = true,
    conclusionAsProperty = false,
  } = config

  const firstHeading = find(tree, {type: 'heading', depth: ROOT_HEADING_DEPTH})
  if (!firstHeading) {
    throw new Error('No heading')
  }
  return splitAtDepth(
    tree,
    ROOT_HEADING_DEPTH,
    {splitDepth, introductionAsProperty, conclusionAsProperty}
  )
}

function splitAtDepth (tree, currentDepth, config) {
  const {
    splitDepth = ROOT_HEADING_DEPTH,
    introductionAsProperty = true,
    conclusionAsProperty = false,
  } = config

  const splitter = new Splitter(currentDepth)
  const hasHeading = !!find(tree, {type: 'heading', depth: currentDepth})
  if (!hasHeading) {
    return tree
  }
  visit(tree, null, (node, index, parent) => splitter.visit(node, index, parent))
  if (conclusionAsProperty) {
    splitter.extractConclusions()
  }
  if (introductionAsProperty) {
    splitter.extractIntroductions()
  }
  if (currentDepth < splitDepth) {
    for (let i = 0; i < splitter.subTrees.length; i++) {
      const newTree = {
        type: 'root',
        children: splitter.subTrees[i].children.children,
      }
      const result = splitAtDepth(newTree, currentDepth + 1, config)
      splitter.subTrees[i].children = result
    }
  }
  return {
    introduction: splitter.introduction,
    trees: splitter.subTrees,
  }
}

function find (tree, {type = 'heading', depth = ROOT_HEADING_DEPTH}) {
  for (let i = 0; i < tree.children.length; i++) {
    if (tree.children[i].type === type && tree.children[i].depth === depth) {
      return tree.children[i]
    }
  }
  return null
}

class Splitter {
  constructor (depth = 1) {
    this.lastIndex = -1
    this.subTrees = []
    this.depth = depth
    this._introduction = {
      type: 'root',
      children: [],
    }
  }
  set introduction (nodes) {
    this._introduction = {type: 'root', children: nodes}
  }
  get introduction () {
    return this._introduction
  }
  extractConclusions () {
    this.subTrees.forEach(splittedPart => {
      const firstHeading = find(splittedPart.children, {
        type: 'heading',
        depth: this.depth + 1,
      })
      let lastHeading = null
      let lastIndex = 0
      for (let i = splittedPart.children.length - 1; i > 0; i--) {
        if (splittedPart.children[i] === 'heading' &&
          splittedPart.children[i].depth === this.depth - 1) {
          lastHeading = splittedPart.children[i]
          lastIndex = i
          break
        }
      }
      if (!lastHeading || lastHeading === firstHeading) {
        return
      }
      splittedPart.conclusion = {
        type: 'root',
        children: splittedPart.children.splice(lastIndex, splittedPart.children.length - lastIndex),
      }
    })
  }
  extractIntroductions () {
    this.subTrees.forEach(
      (subTree) => {
        const firstHeading = find(subTree.children, {
          type: 'heading',
          depth: this.depth + 1,
        })
        if (firstHeading) {
          subTree.introduction = {
            type: 'root',
            children: subTree.children.children.splice(0,
              subTree.children.children.indexOf(firstHeading)),
          }
        }
      }
    )
  }

{'fraction_non_alphanumeric': 0.12224866151100536, 'fraction_numerical': 0.003866745984533016, 'mean_word_length': 2.6020353508302088, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7376455', 'n_tokens_mistral': 2161, 'n_tokens_neox': 2103, 'n_words': 455}