<commit_before>const htmlConvert = require('html-to-text')
// const iconv = require('iconv-lite')
const defaultConfigs = require('../util/checkConfig.js').defaultConfigs
const config = require('../config.js')
const EXCLUDED_KEYS = ['title', 'description', 'summary', 'author', 'link', 'pubDate', 'pubdate', 'date']

function cleanup (source, text, encoding) {
  if (!text) return ''

  let newText = text
  newText = newText.replace(/\*/gi, '')
    .replace(/<(strong|b)>(.*?)<\/(strong|b)>/gi, '**$2**') // Bolded markdown
    .replace(/<(em|i)>(.*?)<(\/(em|i))>/gi, '*$2*') // Italicized markdown
    .replace(/<(u)>(.*?)<(\/(u))>/gi, '__$2__') // Underlined markdown

  newText = htmlConvert.fromString(newText, {
    tables: (source.formatTables !== undefined && typeof source.formatTables === 'boolean' ? source.formatTables : config.feeds.formatTables) === true ? true : [],
    wordwrap: null,
    ignoreHref: true,
    noLinkBrackets: true,
    format: {
      image: node => {
        const isStr = typeof node.attribs.src === 'string'
        let link = isStr ? node.attribs.src.trim() : node.attribs.src
        if (isStr && link.startsWith('//')) link = 'http:' + link
        else if (isStr && !link.startsWith('http://') && !link.startsWith('https://')) link = 'http://' + link

        let exist = true
        const globalExistOption = config.feeds.imgLinksExistence != null ? config.feeds.imgLinksExistence : defaultConfigs.feeds.imgLinksExistence.default // Always a boolean via startup checks
        exist = globalExistOption
        const specificExistOption = source.imgLinksExistence
        exist = typeof specificExistOption !== 'boolean' ? exist : specificExistOption
        if (!exist) return ''

        let image = ''
        const globalPreviewOption = config.feeds.imgPreviews != null ? config.feeds.imgPreviews : defaultConfigs.feeds.imgPreviews.default // Always a boolean via startup checks
        image = globalPreviewOption ? link : `<${link}>`
        const specificPreviewOption = source.imgPreviews
        image = typeof specificPreviewOption !== 'boolean' ? image : specificPreviewOption === true ? link : `<${link}>`

        return image
      }
    }
  })

  newText = newText.replace(/\n\s*\n\s*\n/g, '\n\n') // Replace triple line breaks with double
  const arr = newText.split('\n')
  for (var q = 0; q < arr.length; ++q) arr[q] = arr[q].replace(/\s+$/, '') // Remove trailing spaces
  return arr.join('\n')
}

class FlattenedJSON {
  constructor (data, source, encoding = 'utf-8') {
    this.encoding = encoding.toLowerCase()
    this.source = source
    this.data = data
    this.results = {}
    this.text = ''
    // Populate this.results with a trampoline to avoid stack call exceeded
    this._trampolineIteration(this._iterateOverObject.bind(this), data)
    // Generate the text from this.results
    this._generateText()
  }

  _trampolineIteration (fun, obj, previousKeyNames) {
    for (var key in obj) {
      let val = fun.bind(this)(obj[key], key, previousKeyNames)
      while (typeof val === 'function') {
        val = val()
      }
    }
  }

  _iterateOverObject (item, keyName, previousKeyNames) {
    const keyNameWithPrevious = previousKeyNames ? `${previousKeyNames}_${keyName}` : keyName
    if (Array.isArray(item) || !item || EXCLUDED_KEYS.includes(keyName) || item === this.results[keyNameWithPrevious.toLowerCase()]) return
    if (Object.prototype.toString.call(item) === '[object Object]') return () => this._trampolineIteration(this._iterateOverObject, item, keyNameWithPrevious)
    else this.results[keyNameWithPrevious] = item
  }

  _generateText () {
    let nameHeader = 'PROPERTY NAME'
    let valueHeader = 'VALUE'
    let longestNameLen = 0<commit_msg>Fix colon and emoji conflicts with raw placeholders<commit_after>const htmlConvert = require('html-to-text')
// const iconv = require('iconv-lite')
const defaultConfigs = require('../util/checkConfig.js').defaultConfigs
const config = require('../config.js')
const EXCLUDED_KEYS = ['title', 'description', 'summary', 'author', 'link', 'pubDate', 'pubdate', 'date']

function cleanup (source, text, encoding) {
  if (!text) return ''

  let newText = text
  newText = newText.replace(/\*/gi, '')
    .replace(/<(strong|b)>(.*?)<\/(strong|b)>/gi, '**$2**') // Bolded markdown
    .replace(/<(em|i)>(.*?)<(\/(em|i))>/gi, '*$2*') // Italicized markdown
    .replace(/<(u)>(.*?)<(\/(u))>/gi, '__$2__') // Underlined markdown

  newText = htmlConvert.fromString(newText, {
    tables: (source.formatTables !== undefined && typeof source.formatTables === 'boolean' ? source.formatTables : config.feeds.formatTables) === true ? true : [],
    wordwrap: null,
    ignoreHref: true,
    noLinkBrackets: true,
    format: {
      image: node => {
        const isStr = typeof node.attribs.src === 'string'
        let link = isStr ? node.attribs.src.trim() : node.attribs.src
        if (isStr && link.startsWith('//')) link = 'http:' + link
        else if (isStr && !link.startsWith('http://') && !link.startsWith('https://')) link = 'http://' + link

        let exist = true
        const globalExistOption = config.feeds.imgLinksExistence != null ? config.feeds.imgLinksExistence : defaultConfigs.feeds.imgLinksExistence.default // Always a boolean via startup checks
        exist = globalExistOption
        const specificExistOption = source.imgLinksExistence
        exist = typeof specificExistOption !== 'boolean' ? exist : specificExistOption
        if (!exist) return ''

        let image = ''
        const globalPreviewOption = config.feeds.imgPreviews != null ? config.feeds.imgPreviews : defaultConfigs.feeds.imgPreviews.default // Always a boolean via startup checks
        image = globalPreviewOption ? link : `<${link}>`
        const specificPreviewOption = source.imgPreviews
        image = typeof specificPreviewOption !== 'boolean' ? image : specificPreviewOption === true ? link : `<${link}>`

        return image
      }
    }
  })

  newText = newText.replace(/\n\s*\n\s*\n/g, '\n\n') // Replace triple line breaks with double
  const arr = newText.split('\n')
  for (var q = 0; q < arr.length; ++q) arr[q] = arr[q].replace(/\s+$/, '') // Remove trailing spaces
  return arr.join('\n')
}

class FlattenedJSON {
  constructor (data, source, encoding = 'utf-8') {
    this.encoding = encoding.toLowerCase()
    this.source = source
    this.data = data
    this.results = {}
    this.text = ''
    // Populate this.results with a trampoline to avoid stack call exceeded
    this._trampolineIteration(this._iterateOverObject.bind(this), data)
    // Generate the text from this.results
    this._generateText()
  }

  _trampolineIteration (fun, obj, previousKeyNames) {
    for (var key in obj) {
      let val = fun.bind(this)(obj[key], key, previousKeyNames)
      while (typeof val === 'function') {
        val = val()
      }
    }
  }

  _iterateOverObject (item, keyName, previousKeyNames) {
    const keyNameWithPrevious = (previousKeyNames ? `${previousKeyNames}_${keyName}` : keyName).replace(':', '-') // Replace colons to avoid emoji conflicts
    if (Array.isArray(item) || !item || EXCLUDED_KEYS.includes(keyName) || item === this.results[keyNameWithPrevious.toLowerCase()]) return
    if (Object.prototype.toString.call(item) === '[object Object]') return () => this._trampolineIteration(this._iterateOverObject, item, keyNameWithPrevious)
    else this.results[keyNameWithPrevious] = item
  }

  _generateText () {
    let nameHeader = 'PROPERTY NAME'
    let valueHeader = 'VALUE'
    let longestNameLen = 0
{'fraction_non_alphanumeric': 0.15719372280100224, 'fraction_numerical': 0.0015824871422919688, 'mean_word_length': 3.424737456242707, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7025087', 'n_tokens_mistral': 2334, 'n_tokens_neox': 2308, 'n_words': 552}