<commit_before>
const purifyData = require('./purify')
const processImages = require('./images')
const { pullRepo, pushChanges } = require('./git')

main()

async function main () {
  try {
    // pull down latest version
    // await pullRepo()

    // ensure slugs, etc, are of the correct format
    await purifyData()

    // move images to correct place, generate smaller sizes
    await processImages()

    // commit changes (if any made)
    // await pushChanges()
  } catch (err) {
    console.error(err)
  }
}<commit_msg>chore: uncomment transformer git lines<commit_after>
const purifyData = require('./purify')
const processImages = require('./images')
const { pullRepo, pushChanges } = require('./git')

main()

async function main () {
  try {
    // pull down latest version
    await pullRepo()

    // ensure slugs, etc, are of the correct format
    await purifyData()

    // move images to correct place, generate smaller sizes
    await processImages()

    // commit changes (if any made)
    await pushChanges()
  } catch (err) {
    console.error(err)
  }
}
{'fraction_non_alphanumeric': 0.1211840888066605, 'fraction_numerical': 0.0, 'mean_word_length': 3.0223048327137545, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4971158', 'n_tokens_mistral': 309, 'n_tokens_neox': 310, 'n_words': 108}