<commit_before>'use strict'

const fs = require('fs')
const { join } = require('path')
const { spawn, execFile } = require('child_process')
const rimraf = require('rimraf')

module.exports = args => {
  const url = args.url || args.u
  const branch = args.branch || args.b || 'master'
  const git = args.git || args.g || 'git'
  if (!url) {
    throw new Error('no url given! (e.g. --url https://github/me/myrepo.git)')
  }
  return output =>
    new Promise((resolve, reject) => {
      // Checkout repository
      fs.mkdtemp('penguin-git-clone-', (err, path) => {
        if (err) return reject(err)
        execFile(git, [
          'clone', '--no-checkout',
          '--depth', '1',
          '--branch', branch,
          url, path
        ],
          err => {
            if (err) return reject(err)
            rimraf(join(output, '.git'), err => {
              if (err) return reject(err)
              fs.rename(join(path, '.git'), join(output, '.git'), err => {
                if (err) return reject(err)
                fs.rmdir(path, err => {
                  if (err) return reject(err)
                  resolve()
                })
              })
            })
          })
      })
    })
    .then(() => {
      return new Promise((resolve, reject) => {
        spawn(git, ['add', '.'], { stdio: 'inherit', cwd: output })
          .on('error', reject).on('close', code => {
            if (code !== 0) return reject(new Error(code))
            commit()
          })
        function commit () {
          execFile(git, [
            '-c', 'user.name=Penguin',
            '-c', 'user.email=<>',
            'commit',
            '-m', 'Update content from penguin.js'
          ], { cwd: output }, err => {
            if (err && err.code !== 1) return reject(err)
            push()
          })
        }
        function push () {
          execFile(git, ['push', url, branch], { cwd: output }, err => {
            if (err) return reject(err)
            resolve()
          })
        }
      })
    })
}
<commit_msg>Set higher maxBuffer (avoid stdout maxBuffer exceeded)<commit_after>'use strict'

const fs = require('fs')
const { join } = require('path')
const { spawn, execFile } = require('child_process')
const rimraf = require('rimraf')

module.exports = args => {
  const url = args.url || args.u
  const branch = args.branch || args.b || 'master'
  const git = args.git || args.g || 'git'
  if (!url) {
    throw new Error('no url given! (e.g. --url https://github/me/myrepo.git)')
  }
  return output =>
    new Promise((resolve, reject) => {
      // Checkout repository
      fs.mkdtemp('penguin-git-clone-', (err, path) => {
        if (err) return reject(err)
        execFile(git, {maxBuffer: 1024 * 1024}, [
          'clone', '--no-checkout',
          '--depth', '1',
          '--branch', branch,
          url, path
        ],
          err => {
            if (err) return reject(err)
            rimraf(join(output, '.git'), err => {
              if (err) return reject(err)
              fs.rename(join(path, '.git'), join(output, '.git'), err => {
                if (err) return reject(err)
                fs.rmdir(path, err => {
                  if (err) return reject(err)
                  resolve()
                })
              })
            })
          })
      })
    })
    .then(() => {
      return new Promise((resolve, reject) => {
        spawn(git, ['add', '.'], { stdio: 'inherit', cwd: output })
          .on('error', reject).on('close', code => {
            if (code !== 0) return reject(new Error(code))
            commit()
          })
        function commit () {
          execFile(git, {maxBuffer: 1024 * 1024}, [
            '-c', 'user.name=Penguin',
            '-c', 'user.email=<>',
            'commit',
            '-m', 'Update content from penguin.js'
          ], { cwd: output }, err => {
            if (err && err.code !== 1) return reject(err)
            push()
          })
        }
        function push () {
          execFile(git, {maxBuffer: 1024 * 1024}, ['push', url, branch], { cwd: output }, err => {
            if (err) return reject(err)
            resolve()
          })
        }
      })
    })
}

{'fraction_non_alphanumeric': 0.17468594453662006, 'fraction_numerical': 0.00711068973690448, 'mean_word_length': 1.5684723067559343, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 31, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1123521', 'n_tokens_mistral': 1296, 'n_tokens_neox': 1256, 'n_words': 315}