<commit_before>#!/usr/bin/env node

'use strict'

const output = (function () {
  const ignore = Symbol('ignore')
  const headers = Object.freeze([
    'title',
    'type',
    'redirectTitle',
    ignore,
    'categories',
    ignore,
    'related',
    ignore,
    'links',
    'disambiguationContent',
    'image',
    'abstract',
    'url'
  ])

  return {
    writeRow (fields) {
      process.stdout.write(row(fields).join('\t') + '\n')
    }
  }

  function row (fields) {
    return headers.map(header => escape(fields[header] || ''))
  }

  function escape (text) {
    return text
      .replace(/\\/g, '\\\\')
      .replace(/\n/g, '\\n')
      .replace(/\t/g, '\\t')
  }
})()

const fmt = {
  splitSubwords (title) {
    return title
      .replace(/(mouse)/, '$1 ') // “mouseleave” → “mouse leave”
      .replace(/jQuery/, 'jquery') // don’t split jQuery
      .replace(/([a-z])([A-Z])/g, '$1 $2') // “resolveWith” -> “resolve With”
  },
  dollarize (title) {
    return title
      .replace(/^jQuery./, '$.')
  },
  removeContext (title) {
    return title
      .replace(/^\w+\./, '.') // “jQuery.noop()” → “.noop()”
      .replace(/ \[.+\]$/, '') // “Has Attribute Selector [name]” → “has attribute selector”
      .replace(/ \(.+\)$/, '') // “Element Selector ("element")” → “element selector”
  },
  removeSpecialCharacters (title) {
    return title
      .toLowerCase()
      .replace(/[^a-z]+/g, ' ') // “:gt() Selector” → “gt selector”``
      .trim()
  },
  removeTrailingParens (title) {
    return title
      .replace(/\(\)$/, '')
  },
  removeLeadingDot (title) {
    return title
      .replace(/^\./, '')
  }
}

if (require.main === module) {
  const readFile = require('fs').readFile
  const basename = require('path').basename

  const xpath = require('xpath')
  const DOMParser = require('xmldom').DOMParser

  const url = path => 'https://api.jquery.com/' + basename(path, '.xml') + '/'

  const CATEGORIES = {
    method: 'jQuery methods',
    property: 'jQuery properties',
    selector: 'jQuery selectors'
  }

  const paths = process.argv.slice(2)

  paths.forEach(path => {
    readFile(path, (err, data) => {
      if (err) throw err

      const doc = new DOMParser().parseFromString(data.toString())

      const entries = xpath.select('//entry', doc)

      const title = xpath.select1('./title', entries[0]).textContent
      const entryType = entries[0].getAttribute('type')
<commit_msg>Use promises for reading XML documents<commit_after>#!/usr/bin/env node

'use strict'

const fs = require('fs')
const DOMParser = require('xmldom').DOMParser

const output = (function () {
  const ignore = Symbol('ignore')
  const headers = Object.freeze([
    'title',
    'type',
    'redirectTitle',
    ignore,
    'categories',
    ignore,
    'related',
    ignore,
    'links',
    'disambiguationContent',
    'image',
    'abstract',
    'url'
  ])

  return {
    writeRow (fields) {
      process.stdout.write(row(fields).join('\t') + '\n')
    }
  }

  function row (fields) {
    return headers.map(header => escape(fields[header] || ''))
  }

  function escape (text) {
    return text
      .replace(/\\/g, '\\\\')
      .replace(/\n/g, '\\n')
      .replace(/\t/g, '\\t')
  }
})()

const fmt = {
  splitSubwords (title) {
    return title
      .replace(/(mouse)/, '$1 ') // “mouseleave” → “mouse leave”
      .replace(/jQuery/, 'jquery') // don’t split jQuery
      .replace(/([a-z])([A-Z])/g, '$1 $2') // “resolveWith” -> “resolve With”
  },
  dollarize (title) {
    return title
      .replace(/^jQuery./, '$.')
  },
  removeContext (title) {
    return title
      .replace(/^\w+\./, '.') // “jQuery.noop()” → “.noop()”
      .replace(/ \[.+\]$/, '') // “Has Attribute Selector [name]” → “has attribute selector”
      .replace(/ \(.+\)$/, '') // “Element Selector ("element")” → “element selector”
  },
  removeSpecialCharacters (title) {
    return title
      .toLowerCase()
      .replace(/[^a-z]+/g, ' ') // “:gt() Selector” → “gt selector”``
      .trim()
  },
  removeTrailingParens (title) {
    return title
      .replace(/\(\)$/, '')
  },
  removeLeadingDot (title) {
    return title
      .replace(/^\./, '')
  }
}

function readXMLDoc (path) {
  return new Promise((resolve, reject) => {
    fs.readFile(path, (err, data) => {
      if (err) throw err

      const parser = new DOMParser()
      const xml = data.toString()

      resolve(parser.parseFromString(xml))
    })
  })
}

if (require.main === module) {
  const basename = require('path').basename
  const xpath = require('xpath')

  const url = path => 'https://api.jquery.com/' + basename(path, '.xml') + '/'

  const CATEGORIES = {
    method: 'jQuery methods',
    property: 'jQuery properties',
    selector: 'jQuery selectors'
  }

  process.argv.slice(2).forEach(path => {
    readXMLDoc(path).then(doc => {
      const entries = xpath.select('//entry', doc)

      const title = xpath.select1('./title', entries[0]).textContent
      const entryType = entries[0].getAttribute('type')

{'fraction_non_alphanumeric': 0.20326823435631725, 'fraction_numerical': 0.0027899561578318055, 'mean_word_length': 2.5394922425952045, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7630246', 'n_tokens_mistral': 1707, 'n_tokens_neox': 1674, 'n_words': 372}