<commit_before>/* send bookmarks data.  */

function extractPageText (doc, win) {
  var maybeNodes = [].slice.call(doc.body.childNodes)
  var textNodes = []

  var ignore = 'link, style, script, noscript, .hidden, [class*="-hidden"], .visually-hidden, .visuallyhidden, [role=presentation], [hidden], [style*="display:none"], [style*="display: none"], .ad, .dialog, .modal, select, svg, details:not([open])'

  while(maybeNodes.length) {
    var node = maybeNodes[0]

    // remove the node from the list of nodes to check
    maybeNodes.shift()

    // if the node should be ignored, skip it and all of it's child nodes
    if (node.matches && node.matches(ignore)) {
      continue
    }

    // if the node is a text node, add it to the list of text nodes

    if (node.nodeType === 3) {
      textNodes.push(node)
      continue
    }

    // otherwise, add the node's text nodes to the list of text, and the other child nodes to the list of nodes to check
    var childNodes = node.childNodes
    var cnl = childNodes.length

    for (var i = 0; i < cnl; i++) {
      var childNode = childNodes[i]
      // text node
      if (childNode.nodeType === 3) {
        textNodes.push(childNode)
      } else {
        maybeNodes.unshift(childNode)
      }
    }
  }

  var text = ''

  var tnl = textNodes.length

  // combine the text of all of the accepted text nodes together
  for (var i = 0; i < tnl; i++) {
    text += textNodes[i].textContent + ' '
  }

  // special meta tags

  var mt = doc.head.querySelector('meta[name=description]')

  if (mt) {
    text += ' ' + mt.content
  }

  text = text.trim()

  text = text.replace(/[\n\t]/g, '') // remove useless newlines/tabs that increase filesize

  text = text.replace(/\s{2,}/g, ' ') // collapse multiple spaces into one

  return text
}

function getPageData () {
  /* also parse special metadata: price, rating, location, cooking time */

  var price, rating, location, cookTime

  // pricing

  var priceEl = document.querySelector('[itemprop=price], .price, .offer-price, #priceblock_ourprice, .discounted-price')
  var currencyEl = document.querySelector('[itemprop=priceCurrency], [property=priceCurrency]')

  if (priceEl) {
    price = priceEl.getAttribute('content') || priceEl.textContent
  }

  if (currencyEl) {
    var currency = currencyEl.getAttribute('content') || currencyEl.textContent
  }

  if (!/\d/g.test(price)) { // if the price doesn't contain a number, it probably isn't accurate
    price = undefined
  }

  var currencySymbolMap = {
    'USD': '$',<commit_msg>fix parsing of multi-line text nodes<commit_after>/* send bookmarks data.  */

function extractPageText (doc, win) {
  var maybeNodes = [].slice.call(doc.body.childNodes)
  var textNodes = []

  var ignore = 'link, style, script, noscript, .hidden, [class*="-hidden"], .visually-hidden, .visuallyhidden, [role=presentation], [hidden], [style*="display:none"], [style*="display: none"], .ad, .dialog, .modal, select, svg, details:not([open])'

  while(maybeNodes.length) {
    var node = maybeNodes[0]

    // remove the node from the list of nodes to check
    maybeNodes.shift()

    // if the node should be ignored, skip it and all of it's child nodes
    if (node.matches && node.matches(ignore)) {
      continue
    }

    // if the node is a text node, add it to the list of text nodes

    if (node.nodeType === 3) {
      textNodes.push(node)
      continue
    }

    // otherwise, add the node's text nodes to the list of text, and the other child nodes to the list of nodes to check
    var childNodes = node.childNodes
    var cnl = childNodes.length

    for (var i = 0; i < cnl; i++) {
      var childNode = childNodes[i]
      // text node
      if (childNode.nodeType === 3) {
        textNodes.push(childNode)
      } else {
        maybeNodes.unshift(childNode)
      }
    }
  }

  var text = ''

  var tnl = textNodes.length

  // combine the text of all of the accepted text nodes together
  for (var i = 0; i < tnl; i++) {
    text += textNodes[i].textContent + ' '
  }

  // special meta tags

  var mt = doc.head.querySelector('meta[name=description]')

  if (mt) {
    text += ' ' + mt.content
  }

  text = text.trim()

  text = text.replace(/[\n\t]/g, ' ') // remove useless newlines/tabs that increase filesize

  text = text.replace(/\s{2,}/g, ' ') // collapse multiple spaces into one

  return text
}

function getPageData () {
  /* also parse special metadata: price, rating, location, cooking time */

  var price, rating, location, cookTime

  // pricing

  var priceEl = document.querySelector('[itemprop=price], .price, .offer-price, #priceblock_ourprice, .discounted-price')
  var currencyEl = document.querySelector('[itemprop=priceCurrency], [property=priceCurrency]')

  if (priceEl) {
    price = priceEl.getAttribute('content') || priceEl.textContent
  }

  if (currencyEl) {
    var currency = currencyEl.getAttribute('content') || currencyEl.textContent
  }

  if (!/\d/g.test(price)) { // if the price doesn't contain a number, it probably isn't accurate
    price = undefined
  }

  var currencySymbolMap = {
    'USD': '$',
{'fraction_non_alphanumeric': 0.1354044548651817, 'fraction_numerical': 0.0023446658851113715, 'mean_word_length': 2.974378881987578, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1178477', 'n_tokens_mistral': 1630, 'n_tokens_neox': 1570, 'n_words': 495}