<commit_before>const fs = require('fs')
const path = require('path')
const SimpleMarkdown = require('simple-markdown')
const { parseFrontMatter } = require('./md-loader-utils.js')

const { slugify } = require('./utils')

const levelName = 'l'
const stripEmptyContent = true

// get the menu from assets folder
const menu = require(path.resolve(__dirname, '../src/assets/menu.js'))

// markdown parser (not perfect, but works well enough)
const mdParse = SimpleMarkdown.defaultBlockParse

// // eslint-disable-next-line no-useless-escape
// const yamlBlockPattern = /^(?:\-\-\-)(.*?)(?:\-\-\-|\.\.\.)/s

// where the markdown lives
const intro = '../src/pages'

let objectID = 1
const getObjectID = () => objectID++

const createFolder = (folder) => {
  const dir = path.join(__dirname, '..', folder)
  if (!fs.existsSync(dir)) {
    fs.mkdirSync(dir)
  }
}

const createIndex = (data) => {
  const requiredFields = [ levelName + '0', 'url' ]
  const missingFields = requiredFields.filter(
    (requiredField) => !data[ requiredField ]
  )
  if (missingFields.length) {
    throw new Error(
      `Missing fields for indexing page ${data.url}: ${missingFields.join(
        ', '
      )}`
    )
  }
  return {
    id: getObjectID(),
    [ levelName + '0' ]: null,
    [ levelName + '1' ]: null,
    [ levelName + '2' ]: null,
    [ levelName + '3' ]: null,
    [ levelName + '4' ]: null,
    [ levelName + '5' ]: null,
    [ levelName + '6' ]: null,
    keys: null,
    content: null,
    anchor: null,
    ...data
  }
}

const cleanObject = (item) => {
  const keys = Object.keys(item)
  for (const key in keys) {
    if (item[ keys[ key ] ] === null) {
      delete item[ keys[ key ] ]
    }
  }
  return item
}

const getNextLevel = (item, text) => {
  for (let index = 0; index < 7; ++index) {
    if (item[ levelName + index ] === void 0 || item[ levelName + index ] === text) {
      return index
    }
  }
  return 6 // highest
}

// makes sure there is content before adding to array
const addItem = (entries, item) => {
  if (stripEmptyContent === true && (item.content === null || item.content === '')) {
    return
  }
  entries.push(cleanObject(createIndex(item)))
}

// returns the contents of the associated file
const getFileContents = (mdPath) => {
  const page = path.resolve(__dirname, mdPath)
  return fs.readFileSync(page, {
    encoding: 'utf8'
  })
}

// responsible for collecting the text
const buildParagraph = (content, remaining = '', skip = 0) => {
  const text = []

  const addText = (data) => {
    text.push(data)
  }

  addText(remaining)

  content.forEach((node, index) => {
    if (index >= skip) {
      if (node.type === 'text' || node.type === 'inlineCode') {
        addText(node.content)
      }
      else {
        if (node.content && Array.isArray(node.content)) {
          const data = buildParagraph(node.content)
          addText(data.text)
        }
        else if (node.items && Array.isArray(node.items)) {
          node.items.forEach(leaf => {
            const data = buildParagraph(leaf)
            addText(data.text)
          })
        }
        else {
          if (node.type !== 'image' && node.type !== 'codeBlock') {
            console.log('Unknown node:', node)
          }
        }
      }
    }
  })

  return { text: text.join('') }
}

const processNode = (node, entry) => {
  const text = []
  let type = 'page-content'
  if (Array.isArray(node)) {
    node.forEach(leaf => {
      const data = processNode(leaf)
      text.push(data.text)
    })
  }
  else if (node.type === 'list' && node.items && Array.isArray(node.items)) {
    type = 'page-list'
    node.items.forEach(leaf => {
      const data = buildParagraph(leaf)
      text.push('* ' + data.text)
    })
  }
  else if (node.type === 'paragraph' && Array.isArray(node.content)) {
    if (node.content[ 0 ].type === 'text' && node.content[ 0 ].content.startsWith('#')) {
      let text = ''
      let remaining = ''
      let index = 0
      for (; index < node.content.length; ++index) {
        let content = node.content[ index ].content
        if (Array.isArray(content)) {
          content = buildParagraph(content).text
        }
        const parts = content.split('\n')
        text += parts[ 0 ]
        if (parts.length > 1) {
          parts.shift()
          remaining += parts.join('')
          break
        }
      }
      const level2 = text.indexOf(' ')
      const subheading = text.slice(level2 + 1)
      const level = getNextLevel(entry, subheading)
      const entryItem = {
        [ levelName + (level) ]: subheading,
        anchor: slugify(subheading)
      }
      const data = buildParagraph(node.content, remaining, index + 1)
      entryItem.content = data.text
      return entryItem
    }

    const data = buildParagraph(node.content)
    text.push(data.text)
  }
  else if (node.type === 'heading') {
    type = 'page-heading'
    const data = buildParagraph(node.content)
    const subheading = data.text
    const level = getNextLevel(entry, subheading)
    const id = {
      [ levelName + (level) ]: subheading,
      anchor: slugify(subheading)
    }
    return id
  }
  else if (node.type === 'blockQuote') {
    // type = 'page-quote'
    type = 'page-content'
    const data = processNode(node.content, entry)
    text.push(data.text)
  }
  else {
    if (node.type !== 'table' && node.type !== 'codeBlock' && node.type !== 'newline') {
      const data = buildParagraph(node.content)
      text.push(data.text)
    }
  }
  return { text: text.join(' '), type }
}

const processTip = (val) => {
  const start = val.text.indexOf('\n')
  const end = val.text.lastIndexOf('\n')
  // val.type = 'page-tip'
  val.type = 'page-content'
  val.text = val.text.substr(start + 1, val.text.length - start - (val.text.length - end) - 1)
  return val
}

const processMarkdown = (syntaxTree, entries, entry) => {
  const contents = []
  let type = 'page-content'
  let parent = { ...entry }

  const handleAnchor = (val = null) => {
    const joiner = type === 'page-list' ? '' : ' '
    if (contents.length > 0) {
      const text = contents.join(joiner)
        // .replace(/\n/g, ' ')
        .replace(/<br>/g, '\n')
        .replace(/\|/g, '')
        .replace(/\s\s+/g, ' ')
        .replace(/::: tip/g, '')
        .replace(/---/g, '')
        .replace(/::: warning/g, '')
        .replace(/::: danger/g, '')
        .replace(/:::/g, '')
        .trim()

      // handle text from previous
      addItem(entries, { ...parent, content: text, type })

      // start a new index
      parent = { ...entry, content: '' }

      // clean up contents array
      contents.splice(0, contents.length)
    }
  }

  syntaxTree.forEach((node, index) => {
    let val = processNode(node, parent)

    if (val.text !== void 0 && val.text !== '') {
      // don't accept components embedded into the page
      if (val.text.charAt(0) !== '<' && val.text.charAt(val.text.length - 1) !== '>') {
        // look for tips
        if (val.text.startsWith(':::') && val.text.endsWith(':::')) {
          val = processTip(val)
        }

        console.log(val)
        if (val.anchor || type !== val.type) {
          handleAnchor(val)
          contents.push(val.text)
        }
        else {
          contents.push(val.text)
        }

        type = val.type
      }
    }
  })

  // handle last bits on the page
  handleAnchor()
}

const processPage = (page, entry, entries) => {
  const md = getFileContents(page)
  const frontMatter = parseFrontMatter(md)
  let keys = null
<commit_msg>feat(docs): better tip handling for search parser<commit_after>const fs = require('fs')
const path = require('path')
const SimpleMarkdown = require('simple-markdown')
const { parseFrontMatter } = require('./md-loader-utils.js')

const { slugify } = require('./utils')

const levelName = 'l'
const stripEmptyContent = true

// get the menu from assets folder
const menu = require(path.resolve(__dirname, '../src/assets/menu.js'))

// markdown parser (not perfect, but works well enough)
const mdParse = SimpleMarkdown.defaultBlockParse

// // eslint-disable-next-line no-useless-escape
// const yamlBlockPattern = /^(?:\-\-\-)(.*?)(?:\-\-\-|\.\.\.)/s

// where the markdown lives
const intro = '../src/pages'

let objectID = 1
const getObjectID = () => objectID++

const createFolder = (folder) => {
  const dir = path.join(__dirname, '..', folder)
  if (!fs.existsSync(dir)) {
    fs.mkdirSync(dir)
  }
}

const createIndex = (data) => {
  const requiredFields = [ levelName + '0', 'url' ]
  const missingFields = requiredFields.filter(
    (requiredField) => !data[ requiredField ]
  )
  if (missingFields.length) {
    throw new Error(
      `Missing fields for indexing page ${data.url}: ${missingFields.join(
        ', '
      )}`
    )
  }
  return {
    id: getObjectID(),
    [ levelName + '0' ]: null,
    [ levelName + '1' ]: null,
    [ levelName + '2' ]: null,
    [ levelName + '3' ]: null,
    [ levelName + '4' ]: null,
    [ levelName + '5' ]: null,
    [ levelName + '6' ]: null,
    keys: null,
    content: null,
    anchor: null,
    ...data
  }
}

const cleanObject = (item) => {
  const keys = Object.keys(item)
  for (const key in keys) {
    if (item[ keys[ key ] ] === null) {
      delete item[ keys[ key ] ]
    }
  }
  return item
}

const getNextLevel = (item, text) => {
  for (let index = 0; index < 7; ++index) {
    if (item[ levelName + index ] === void 0 || item[ levelName + index ] === text) {
      return index
    }
  }
  return 6 // highest
}

// makes sure there is content before adding to array
const addItem = (entries, item) => {
  if (stripEmptyContent === true && (item.content === null || item.content === '')) {
    return
  }
  entries.push(cleanObject(createIndex(item)))
}

// returns the contents of the associated file
const getFileContents = (mdPath) => {
  const page = path.resolve(__dirname, mdPath)
  return fs.readFileSync(page, {
    encoding: 'utf8'
  })
}

// responsible for collecting the text
const buildParagraph = (content, remaining = '', skip = 0) => {
  const text = []

  const addText = (data) => {
    text.push(data)
  }

  addText(remaining)

  content.forEach((node, index) => {
    if (index >= skip) {
      if (node.type === 'text' || node.type === 'inlineCode') {
        addText(node.content)
      }
      else {
        if (node.content && Array.isArray(node.content)) {
          const data = buildParagraph(node.content)
          addText(data.text)
        }
        else if (node.items && Array.isArray(node.items)) {
          node.items.forEach(leaf => {
            const data = buildParagraph(leaf)
            addText(data.text)
          })
        }
        else {
          if (node.type !== 'image' && node.type !== 'codeBlock') {
            console.log('Unknown node:', node)
          }
        }
      }
    }
  })

  return { text: text.join('') }
}

const processTip = (node, entry) => {
  const val = processNode(node, entry)
  if (val.text.startsWith(':::') && val.text.endsWith(':::') !== true) {
    val.needMore = true
  }
  return val
}

const processNode = (node, entry) => {
  const text = []
  let type = 'page-content'
  if (Array.isArray(node)) {
    node.forEach(leaf => {
      const data = processNode(leaf)
      text.push(data.text)
    })
  }
  else if (node.type === 'list' && node.items && Array.isArray(node.items)) {
    type = 'page-list'
    node.items.forEach(leaf => {
      const data = buildParagraph(leaf)
      text.push('* ' + data.text)
    })
  }
  else if (node.type === 'paragraph' && Array.isArray(node.content)) {
    if (node.content[ 0 ].type === 'text' && node.content[ 0 ].content.startsWith('#')) {
      let text = ''
      let remaining = ''
      let index = 0
      for (; index < node.content.length; ++index) {
        let content = node.content[ index ].content
        if (Array.isArray(content)) {
          content = buildParagraph(content).text
        }
        const parts = content.split('\n')
        text += parts[ 0 ]
        if (parts.length > 1) {
          parts.shift()
          remaining += parts.join('')
          break
        }
      }
      const level2 = text.indexOf(' ')
      const subheading = text.slice(level2 + 1)
      const level = getNextLevel(entry, subheading)
      const entryItem = {
        [ levelName + (level) ]: subheading,
        anchor: slugify(subheading)
      }
      const data = buildParagraph(node.content, remaining, index + 1)
      entryItem.content = data.text
      return entryItem
    }

    const data = buildParagraph(node.content)
    text.push(data.text)
  }
  else if (node.type === 'heading') {
    type = 'page-heading'
    const data = buildParagraph(node.content)
    const subheading = data.text
    const level = getNextLevel(entry, subheading)
    const id = {
      [ levelName + (level) ]: subheading,
      anchor: slugify(subheading)
    }
    return id
  }
  else if (node.type === 'blockQuote') {
    type = 'page-content'
    const data = processNode(node.content, entry)
    text.push(data.text)
  }
  else {
    if (node.type !== 'table' && node.type !== 'codeBlock' && node.type !== 'newline') {
      const data = buildParagraph(node.content)
      text.push(data.text)
    }
  }
  return { text: text.join(' '), type }
}

const extractTip = (val) => {
  const start = val.text.indexOf('\n')
  const end = val.text.lastIndexOf('\n')
  val.type = 'page-content'
  val.text = val.text.substr(start + 1, val.text.length - start - (val.text.length - end) - 1)
  return val
}

const processMarkdown = (syntaxTree, entries, entry) => {
  const contents = []
  let type = 'page-content'
  let parent = { ...entry }

  const handleAnchor = (val = null) => {
    const joiner = type === 'page-list' ? '' : ' '
    if (contents.length > 0) {
      const text = contents.join(joiner)
        // .replace(/\n/g, ' ')
        .replace(/<br>/g, '\n')
        .replace(/\|/g, '')
        .replace(/\s\s+/g, ' ')
        .replace(/---/g, '')
        .trim()

      // handle text from previous
      addItem(entries, { ...parent, content: text, type })

      // start a new index
      parent = { ...entry, content: '' }

      // clean up contents array
      contents.splice(0, contents.length)
    }
  }

  let tipData
  syntaxTree.forEach((node, index) => {
    let val = processNode(node, parent)

    if (val.text !== void 0 && val.text !== '') {
      // don't accept components embedded into the page
      if (val.text.charAt(0) !== '<' && val.text.endsWith('/>') === false) {
        if (tipData) {
          if (val.text.endsWith(':::')) {
            // we have ending tip
            val.text = tipData.text + '\n' + val.text
            tipData = void 0
          }
        }
        else if (val.text.startsWith(':::')) {
          if (!val.text.endsWith(':::')) {
            // we need more data
            tipData = val
          }
          else {
            if (tipData !== void 0) {
              tipData += '\n' + val.text
            }
          }
        }

        if (tipData === void 0) {
          // look for tips
          if (val.text.startsWith(':::') && val.text.endsWith(':::')) {
            val = extractTip(val)
          }

          if (val.anchor || type !== val.type) {
            handleAnchor(val)
            contents.push(val.text)
          }
          else {
            contents.push(val.text)
          }

          type = val.type
        }
      }
    }
  })

  // handle last bits on the page
  handleAnchor()
}

const processPage = (page, entry, entries) => {
  const md = getFileContents(page)
  const frontMatter = parseFrontMatter(md)
  let keys = null

{'fraction_non_alphanumeric': 0.16563516876156448, 'fraction_numerical': 0.0040834556243220825, 'mean_word_length': 2.2178197495380827, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2444093', 'n_tokens_mistral': 5101, 'n_tokens_neox': 5028, 'n_words': 1214}