<commit_before>var url     = require('url');
var request = require('request');
var _       = require('underscore');
var sig     = require('./sig');


/**
 * Downloads a video after retrieving video info.
 *
 * @param {String} link
 * @param {Object} options
 * @param {ReadableStream} stream
 * @param {Object} info
 * @return {ReadableStream}
 */
module.exports = function download(link, options, stream, info) {
  var formats = info.formats;
  if (options.filter) {
    formats = formats.filter(options.filter);
    if (formats.length === 0) {
      stream.emit('error',
        new Error('no formats found with custom filter'));
      return;
    }
  }

  var format;
  switch (options.quality) {
    case 'highest':
      format = formats[0];
      break;

    case 'lowest':
      format = formats[formats.length - 1];
      break;

    default:
      format = formats.filter(function(format) {
        return format.itag === '' + options.quality;
      })[0];

  }

  if (!format) {
    stream.emit('error', new Error('No such format found'));
    return;
  } else if (format.rtmp) {
    stream.emit('error', new Error('rtmp protocol not supported'));
    return;
  }

  format = _.clone(format);
  sig.get(link, info, format, function(err, format, signature) {
    var decodedUrl;
    if (format.url) {
      decodedUrl = format.url;
    } else if (format.stream) {
      if (format.conn) {
        decodedUrl = format.conn;
        if (decodedUrl[decodedUrl.length - 1] !== '/') {
          decodedUrl += '/';
        }
        decodedUrl += format.stream;
      } else {
        decodedUrl = format.stream;
      }
    } else {
      stream.emit('error', new Error('download url not found'));
      return;
    }

    try {
      decodedUrl = decodeURIComponent(decodedUrl);
    } catch (err) {
      stream.emit('error', err);
      return;
    }

    // Make some adjustments to the final url.
    var parsedUrl = url.parse(decodedUrl, true);
    delete parsedUrl.search;
    var query = parsedUrl.query;
    //delete query.dur;
    //delete query.clen;
    //delete query.gir;
    //delete query.lmt;
    query.ratebypass = 'yes';
    //query.requiressl = 'yes';
    //query.fexp = info.fexp.join(',');
    //query.fexp = '935644,906399,942000,945000,916807,929315,937417,913434,936910,936913,934022';

    // Update `sparams`.
    //query.sparams += ',ratebypass,requiressl';
    //var sparams = query.sparams.split(',');
    //delete query.sparams;
    //query.sparams = Object.keys(query).sort().join(',');
    query.signature = signature;

    var requestOptions = _.clone(options);
    delete requestOptions.quality;
    delete requestOptions.range;
    delete requestOptions.filter;

    requestOptions.url = url.format(parsedUrl);

    // Start downloading the video.
    var res = request(requestOptions);
    res.on('response', function(res) {
      if (res.statusCode !== 200) {
        stream.emit('error', new Error('status code ' + res.statusCode));
        return;<commit_msg>more cleanup<commit_after>var url     = require('url');
var request = require('request');
var _       = require('underscore');
var sig     = require('./sig');


/**
 * Downloads a video after retrieving video info.
 *
 * @param {String} link
 * @param {Object} options
 * @param {ReadableStream} stream
 * @param {Object} info
 * @return {ReadableStream}
 */
module.exports = function download(link, options, stream, info) {
  var formats = info.formats;
  if (options.filter) {
    formats = formats.filter(options.filter);
    if (formats.length === 0) {
      stream.emit('error',
        new Error('no formats found with custom filter'));
      return;
    }
  }

  var format;
  switch (options.quality) {
    case 'highest':
      format = formats[0];
      break;

    case 'lowest':
      format = formats[formats.length - 1];
      break;

    default:
      format = formats.filter(function(format) {
        return format.itag === '' + options.quality;
      })[0];

  }

  if (!format) {
    stream.emit('error', new Error('No such format found'));
    return;
  } else if (format.rtmp) {
    stream.emit('error', new Error('rtmp protocol not supported'));
    return;
  }

  format = _.clone(format);
  sig.get(link, info, format, function(err, format, signature) {
    var decodedUrl;
    if (format.url) {
      decodedUrl = format.url;
    } else if (format.stream) {
      if (format.conn) {
        decodedUrl = format.conn;
        if (decodedUrl[decodedUrl.length - 1] !== '/') {
          decodedUrl += '/';
        }
        decodedUrl += format.stream;
      } else {
        decodedUrl = format.stream;
      }
    } else {
      stream.emit('error', new Error('download url not found'));
      return;
    }

    try {
      decodedUrl = decodeURIComponent(decodedUrl);
    } catch (err) {
      stream.emit('error', err);
      return;
    }

    // Make some adjustments to the final url.
    var parsedUrl = url.parse(decodedUrl, true);
    delete parsedUrl.search;
    var query = parsedUrl.query;
    query.ratebypass = 'yes';
    query.signature = signature;

    var requestOptions = _.clone(options);
    delete requestOptions.quality;
    delete requestOptions.range;
    delete requestOptions.filter;

    requestOptions.url = url.format(parsedUrl);

    // Start downloading the video.
    var res = request(requestOptions);
    res.on('response', function(res) {
      if (res.statusCode !== 200) {
        stream.emit('error', new Error('status code ' + res.statusCode));
        return;
{'fraction_non_alphanumeric': 0.14492491405825944, 'fraction_numerical': 0.01483625836801158, 'mean_word_length': 2.6107119529719136, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2919845', 'n_tokens_mistral': 1778, 'n_tokens_neox': 1700, 'n_words': 411}