<commit_before>import { get } from '../fetch/ajax'
import { merge } from '../util/core'

const cached = {}

function walkFetchEmbed ({ step = 0, embedTokens, compile, fetch }, cb) {
  const token = embedTokens[step]

  if (!token) {
    return cb({})
  }

  const next = text => {
    let embedToken
    if (text) {
      if (token.embed.type === 'markdown') {
        embedToken = compile.lexer(text)
      } else if (token.embed.type === 'code') {
        embedToken = compile.lexer(
          '```' +
            token.embed.lang +
            '\n' +
            text.replace(/`/g, '@DOCSIFY_QM@') +
            '\n```\n'
        )
      }
    }
    cb({ token, embedToken })
    walkFetchEmbed({ step: ++step, compile, embedTokens, fetch }, cb)
  }

  if (process.env.SSR) {
    fetch(token.embed.url).then(next)
  } else {
    get(token.embed.url).then(next)
  }
}

export function prerenderEmbed ({ compiler, raw = '', fetch }, done) {
  let hit
  if ((hit = cached[raw])) {
    return done(hit)
  }

  const compile = compiler._marked
  let tokens = compile.lexer(raw)
  const embedTokens = []
  const linkRE = compile.InlineLexer.rules.link
  const links = tokens.links

  tokens.forEach((token, index) => {
    if (token.type === 'paragraph') {
      token.text = token.text.replace(
        new RegExp(linkRE, 'g'),
        (src, filename, href, title) => {
          const embed = compiler.compileEmbed(href, title)

          if (embed) {
            if (embed.type === 'markdown' || embed.type === 'code') {
              embedTokens.push({
                index,
                embed
              })
            }
            return embed.code
          }

          return src
        }
      )
    }
  })

  let moveIndex = 0
  walkFetchEmbed({ compile, embedTokens, fetch }, ({ embedToken, token }) => {
    if (token) {
      const index = token.index + moveIndex

      merge(links, embedToken.links)

      tokens = tokens<commit_msg>fix(embed): broken in IE, fixed #389, fixed #391<commit_after>import { get } from '../fetch/ajax'
import { merge } from '../util/core'

const cached = {}

function walkFetchEmbed ({ step = 0, embedTokens, compile, fetch }, cb) {
  const token = embedTokens[step]

  if (!token) {
    return cb({})
  }

  const next = text => {
    let embedToken
    if (text) {
      if (token.embed.type === 'markdown') {
        embedToken = compile.lexer(text)
      } else if (token.embed.type === 'code') {
        embedToken = compile.lexer(
          '```' +
            token.embed.lang +
            '\n' +
            text.replace(/`/g, '@DOCSIFY_QM@') +
            '\n```\n'
        )
      }
    }
    cb({ token, embedToken })
    walkFetchEmbed({ step: ++step, compile, embedTokens, fetch }, cb)
  }

  if (process.env.SSR) {
    fetch(token.embed.url).then(next)
  } else {
    get(token.embed.url).then(next)
  }
}

export function prerenderEmbed ({ compiler, raw = '', fetch }, done) {
  let hit
  if ((hit = cached[raw])) {
    return done(hit)
  }

  const compile = compiler._marked
  let tokens = compile.lexer(raw)
  const embedTokens = []
  const linkRE = compile.InlineLexer.rules.link
  const links = tokens.links

  tokens.forEach((token, index) => {
    if (token.type === 'paragraph') {
      token.text = token.text.replace(
        new RegExp(linkRE.source, 'g'),
        (src, filename, href, title) => {
          const embed = compiler.compileEmbed(href, title)

          if (embed) {
            if (embed.type === 'markdown' || embed.type === 'code') {
              embedTokens.push({
                index,
                embed
              })
            }
            return embed.code
          }

          return src
        }
      )
    }
  })

  let moveIndex = 0
  walkFetchEmbed({ compile, embedTokens, fetch }, ({ embedToken, token }) => {
    if (token) {
      const index = token.index + moveIndex

      merge(links, embedToken.links)

      tokens = tokens
{'fraction_non_alphanumeric': 0.14972180070814364, 'fraction_numerical': 0.0025290844714213456, 'mean_word_length': 1.9581151832460733, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5291421', 'n_tokens_mistral': 1267, 'n_tokens_neox': 1220, 'n_words': 290}