<commit_before>var url      = require('url');
var util     = require('./util');
var crequest = require('./crequest');
var JStream  = require('jstream');


/**
 * Gets signature for formats.
 *
 * @param {String} link
 * @param {Object} info
 * @param {Function(Error, String)} callback
 */
exports.get = function(link, info, callback) {
  if (info.use_cipher_signature) {
    crequest(link, function(err, body) {
      if (err) return callback(err);

      var jsonStr = util.between(body, 'ytplayer.config = ', '</script>');
      if (!jsonStr) {
        return callback(new Error('could not find `ytplayer.config`'));
      }

      var jstream = new JStream();
      var ended = false;
      jstream.on('data', function(config) {
        ended = true;
        jstream.pause();

        var newFormats = util.parseFormats(config.args);
        var html5playerfile = 'http:' + config.assets.js;
        crequest(html5playerfile, function(err, body) {
          if (err) return callback(err);

          var tokens = exports.extractActions(body);
          if (!tokens) {
            callback(
              new Error('could not extract signature deciphering actions'));
            return;
          }

          info.formats = info.formats.map(function(format) {
            var newFormat = newFormats
              .filter(function(f) { return f.itag === format.itag; })[0];
            if (newFormat && newFormat.s) {
              format = newFormat;
            }
            var sig = exports.decipher(tokens, format.s);
            format.url = exports.getDownloadURL(format, sig);
            return format;
          });

          callback(null, info);
        });
      });
      jstream.on('error', function(err) {
        if (ended) { return; }
        callback(
          new Error('could not parse `ytplayer.config`: ' + err.message));
      });
      jstream.end(jsonStr);
    });

  } else {
    info.formats.forEach(function(format) {
      var sig = format.sig || '';
      format.url = exports.getDownloadURL(format, sig);
    });
    callback(null, info);
  }
};


/**
 * @param {Object} format
 * @param {Array.<String>} tokens
 */
exports.getDownloadURL = function(format, sig) {
  var decodedUrl;
  if (format.url) {
    decodedUrl = format.url;
  } else if (format.stream) {
    if (format.conn) {
      decodedUrl = format.conn;
      if (decodedUrl[decodedUrl.length - 1] !== '/') {
        decodedUrl += '/';
      }
      decodedUrl += format.stream;
    } else {
      decodedUrl = format.stream;
    }
  } else {
    console.warn('download url not found for itag ' + format.itag);
    return;
  }

  try {
    decodedUrl = decodeURIComponent(decodedUrl);
  } catch (err) {
    console.warn('could not decode url: ' + err.message);
    return;
  }

  // Make some adjustments to the final url.
  var parsedUrl = url.parse(decodedUrl, true);

  // Deleting the `search` part is necessary otherwise changes to
  // `query` won't reflect when running `url.format()`
  delete parsedUrl.search;

  var query = parsedUrl.query;
  query.ratebypass = 'yes';
  if (sig) {
    query.signature = sig;
  }

  return url.format(parsedUrl);
};


/**
 * Decipher a signature based on action tokens.
 *
 * @param {Array.<String>} tokens
 * @param {String} sig
 * @return {String}
 */
exports.decipher = function(tokens, sig) {
  sig = sig.split('');
  var pos;
  for (var i = 0, len = tokens.length; i < len; i++) {
    var token = tokens[i];
    switch (token[0]) {
    case 'r':
      sig = sig.reverse();
      break;
    case 'w':
      pos = ~~token.slice(1);
      sig = swapHeadAndPosition(sig, pos);
      break;
    case 's':
      pos = ~~token.slice(1);
      sig = sig.slice(pos);
      break;
    case 'p':
      pos = ~~token.slice(1);
      sig.splice(0, pos);
      break;
    }
  }
  return sig.join('');
};


/**
 * Swaps the first element of an array with one of given position.
 *
 * @param {Array.<Object>} arr
 * @param {Number} position
 * @return {Array.<Object>}
 */
function swapHeadAndPosition(arr, position) {
  var first = arr[0];
  arr[0] = arr[position % arr.length];
  arr[position] = first;
  return arr;
}


var jsvarStr = '[a-zA-Z_][a-zA-Z_0-9]*';
var reverseStr = ':function\\(a\\)\\{' +
  '(?:return )?a\\.reverse\\(\\)' +
'\\}';
var sliceStr = ':function\\(a,b\\)\\{' +
  'return a\\.slice\\(b\\)' +
'\\}';
var spliceStr = ':function\\(a,b\\)\\{' +
  'a\\.splice\\(0,b\\)' +
'\\}';
var swapStr = ':function\\(a,b\\)\\{' +
  'var c=a\\[0\\];a\\[0\\]=a\\[b%a\\.length\\];a\\[b\\]=c(?:;return a)?' +
'\\}';
var actionsRegexp = new RegExp(
  'var (' + jsvarStr + ')=\\{((?:(?:' +
    jsvarStr + reverseStr + '|' +
    jsvarStr + sliceStr   + '|' +
    jsvarStr + spliceStr  + '|' +
    jsvarStr + swapStr +
  '),?)+)\\};' +
  'function \\w+\\(a\\)\\{' +
    'a=a\\.split\\(""\\);' +
    '((?:(?:a=)?\\1\\.' + jsvarStr + '\\(a,\\d+\\);)+)' +
    'return a\\.join\\(""\\)' +
  '\\}'
);<commit_msg>use `jsvarStr` in one more spot<commit_after>var url      = require('url');
var util     = require('./util');
var crequest = require('./crequest');
var JStream  = require('jstream');


/**
 * Gets signature for formats.
 *
 * @param {String} link
 * @param {Object} info
 * @param {Function(Error, String)} callback
 */
exports.get = function(link, info, callback) {
  if (info.use_cipher_signature) {
    crequest(link, function(err, body) {
      if (err) return callback(err);

      var jsonStr = util.between(body, 'ytplayer.config = ', '</script>');
      if (!jsonStr) {
        return callback(new Error('could not find `ytplayer.config`'));
      }

      var jstream = new JStream();
      var ended = false;
      jstream.on('data', function(config) {
        ended = true;
        jstream.pause();

        var newFormats = util.parseFormats(config.args);
        var html5playerfile = 'http:' + config.assets.js;
        crequest(html5playerfile, function(err, body) {
          if (err) return callback(err);

          var tokens = exports.extractActions(body);
          if (!tokens) {
            callback(
              new Error('could not extract signature deciphering actions'));
            return;
          }

          info.formats = info.formats.map(function(format) {
            var newFormat = newFormats
              .filter(function(f) { return f.itag === format.itag; })[0];
            if (newFormat && newFormat.s) {
              format = newFormat;
            }
            var sig = exports.decipher(tokens, format.s);
            format.url = exports.getDownloadURL(format, sig);
            return format;
          });

          callback(null, info);
        });
      });
      jstream.on('error', function(err) {
        if (ended) { return; }
        callback(
          new Error('could not parse `ytplayer.config`: ' + err.message));
      });
      jstream.end(jsonStr);
    });

  } else {
    info.formats.forEach(function(format) {
      var sig = format.sig || '';
      format.url = exports.getDownloadURL(format, sig);
    });
    callback(null, info);
  }
};


/**
 * @param {Object} format
 * @param {Array.<String>} tokens
 */
exports.getDownloadURL = function(format, sig) {
  var decodedUrl;
  if (format.url) {
    decodedUrl = format.url;
  } else if (format.stream) {
    if (format.conn) {
      decodedUrl = format.conn;
      if (decodedUrl[decodedUrl.length - 1] !== '/') {
        decodedUrl += '/';
      }
      decodedUrl += format.stream;
    } else {
      decodedUrl = format.stream;
    }
  } else {
    console.warn('download url not found for itag ' + format.itag);
    return;
  }

  try {
    decodedUrl = decodeURIComponent(decodedUrl);
  } catch (err) {
    console.warn('could not decode url: ' + err.message);
    return;
  }

  // Make some adjustments to the final url.
  var parsedUrl = url.parse(decodedUrl, true);

  // Deleting the `search` part is necessary otherwise changes to
  // `query` won't reflect when running `url.format()`
  delete parsedUrl.search;

  var query = parsedUrl.query;
  query.ratebypass = 'yes';
  if (sig) {
    query.signature = sig;
  }

  return url.format(parsedUrl);
};


/**
 * Decipher a signature based on action tokens.
 *
 * @param {Array.<String>} tokens
 * @param {String} sig
 * @return {String}
 */
exports.decipher = function(tokens, sig) {
  sig = sig.split('');
  var pos;
  for (var i = 0, len = tokens.length; i < len; i++) {
    var token = tokens[i];
    switch (token[0]) {
    case 'r':
      sig = sig.reverse();
      break;
    case 'w':
      pos = ~~token.slice(1);
      sig = swapHeadAndPosition(sig, pos);
      break;
    case 's':
      pos = ~~token.slice(1);
      sig = sig.slice(pos);
      break;
    case 'p':
      pos = ~~token.slice(1);
      sig.splice(0, pos);
      break;
    }
  }
  return sig.join('');
};


/**
 * Swaps the first element of an array with one of given position.
 *
 * @param {Array.<Object>} arr
 * @param {Number} position
 * @return {Array.<Object>}
 */
function swapHeadAndPosition(arr, position) {
  var first = arr[0];
  arr[0] = arr[position % arr.length];
  arr[position] = first;
  return arr;
}


var jsvarStr = '[a-zA-Z_][a-zA-Z_0-9]*';
var reverseStr = ':function\\(a\\)\\{' +
  '(?:return )?a\\.reverse\\(\\)' +
'\\}';
var sliceStr = ':function\\(a,b\\)\\{' +
  'return a\\.slice\\(b\\)' +
'\\}';
var spliceStr = ':function\\(a,b\\)\\{' +
  'a\\.splice\\(0,b\\)' +
'\\}';
var swapStr = ':function\\(a,b\\)\\{' +
  'var c=a\\[0\\];a\\[0\\]=a\\[b%a\\.length\\];a\\[b\\]=c(?:;return a)?' +
'\\}';
var actionsRegexp = new RegExp(
  'var (' + jsvarStr + ')=\\{((?:(?:' +
    jsvarStr + reverseStr + '|' +
    jsvarStr + sliceStr   + '|' +
    jsvarStr + spliceStr  + '|' +
    jsvarStr + swapStr +
  '),?)+)\\};' +
  'function ' + jsvarStr + '\\(a\\)\\{' +
    'a=a\\.split\\(""\\);' +
    '((?:(?:a=)?\\1\\.' + jsvarStr + '\\(a,\\d+\\);)+)' +
    'return a\\.join\\(""\\)' +
  '\\}'
);
{'fraction_non_alphanumeric': 0.19126840088727565, 'fraction_numerical': 0.003629764065335753, 'mean_word_length': 2.308539026017345, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3888584', 'n_tokens_mistral': 3408, 'n_tokens_neox': 3364, 'n_words': 754}