<commit_before>'use strict'

const { execFile } = require('child_process')
const streamify = require('streamify')
const request = require('request')
const hms = require('hh-mm-ss')
const http = require('http')
const url = require('url')

const {
  isYouTubeRegex,
  isWin,
  formatDuration,
  has,
  isString
} = require('./util')

let ytdlBinary = require('./get-binary')()

const TEN_MEGABYTES = 1000 * 1000 * 10

const execFileOpts = { maxBuffer: TEN_MEGABYTES }

function youtubeDl (args, options, cb) {
  execFile(ytdlBinary, args, { ...execFileOpts, ...options }, function done (
    err,
    stdout,
    stderr
  ) {
    if (err) return cb(err)
    return cb(null, stdout.trim().split(/\r?\n/))
  })
}

/**
 * Processes data
 *
 * @param {Object} data
 * @param {Object} options
 * @param {Object} stream<commit_msg>fix: add access to ChildProcess from outside (#263)<commit_after>'use strict'

const { execFile } = require('child_process')
const streamify = require('streamify')
const request = require('request')
const hms = require('hh-mm-ss')
const http = require('http')
const url = require('url')

const {
  isYouTubeRegex,
  isWin,
  formatDuration,
  has,
  isString
} = require('./util')

let ytdlBinary = require('./get-binary')()

const TEN_MEGABYTES = 1000 * 1000 * 10

const execFileOpts = { maxBuffer: TEN_MEGABYTES }

function youtubeDl (args, options, cb) {
  return execFile(
    ytdlBinary,
    args,
    { ...execFileOpts, ...options },
    function done (err, stdout, stderr) {
      if (err) return cb(err)
      return cb(null, stdout.trim().split(/\r?\n/))
    }
  )
}

/**
 * Processes data
 *
 * @param {Object} data
 * @param {Object} options
 * @param {Object} stream
{'fraction_non_alphanumeric': 0.15520282186948853, 'fraction_numerical': 0.01352145796590241, 'mean_word_length': 3.7409470752089136, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4964867', 'n_tokens_mistral': 615, 'n_tokens_neox': 586, 'n_words': 150}