<commit_before>#!/usr/bin/env node
const fs = require('fs')
const yaml = require('js-yaml')
const marked = require('marked')

// run cli
if (process.mainModule === module) {
  const args = process.argv.slice(2)
  const fromFile = args[0]
  const toFile = args[1] || '-'
  if (!fromFile || fromFile === '-h' || fromFile === '--help') {
    console.error(
      'usage: node convert-data-file.js <from-file.{md,yaml}> [to-file]',
    )
    process.exit(1)
  }
  convertDataFile({fromFile, toFile})
}

// exported module
module.exports = convertDataFile
function convertDataFile({fromFile, toFile}) {
  const contents = fs.readFileSync(fromFile, 'utf-8')
  let output = contents

  const fileType = fromFile.split('.').slice(-1)[0]
  switch (fileType) {
    case 'md':
      output = processMarkdown(contents)
      break
    case 'yaml':
      output = processYaml(contents)
      break
    default:
      throw new Error(
        `unexpected filetype "${fileType}; expected "md" or "yaml"`,
      )
  }

  output = output + '\n'

  const outStream = toFile === '-'
    ? process.stdout<commit_msg>remove trailing commas<commit_after>#!/usr/bin/env node
const fs = require('fs')
const yaml = require('js-yaml')
const marked = require('marked')

// run cli
if (process.mainModule === module) {
  const args = process.argv.slice(2)
  const fromFile = args[0]
  const toFile = args[1] || '-'
  if (!fromFile || fromFile === '-h' || fromFile === '--help') {
    console.error(
      'usage: node convert-data-file.js <from-file.{md,yaml}> [to-file]'
    )
    process.exit(1)
  }
  convertDataFile({fromFile, toFile})
}

// exported module
module.exports = convertDataFile
function convertDataFile({fromFile, toFile}) {
  const contents = fs.readFileSync(fromFile, 'utf-8')
  let output = contents

  const fileType = fromFile.split('.').slice(-1)[0]
  switch (fileType) {
    case 'md':
      output = processMarkdown(contents)
      break
    case 'yaml':
      output = processYaml(contents)
      break
    default:
      throw new Error(
        `unexpected filetype "${fileType}; expected "md" or "yaml"`
      )
  }

  output = output + '\n'

  const outStream = toFile === '-'
    ? process.stdout
{'fraction_non_alphanumeric': 0.16007276034561163, 'fraction_numerical': 0.006366530241018645, 'mean_word_length': 3.1431261770244823, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4305205', 'n_tokens_mistral': 733, 'n_tokens_neox': 717, 'n_words': 175}