<commit_before>var util      = require('util');
var UrlFormat = require('url').format;
var fetch     = require('node-fetch');
var cheerio   = require('cheerio');

function token (str, key) {
  function RL(num, str) {
    for (var i = 0; i < str.length - 2; i += 3) {
      var d = str.charAt(i + 2);
      d = d >= 'a' ? d.charCodeAt(0) - 87 : Number(d);
      d = str.charAt(i + 1) === '+' ? num >>> d : num << d;
      num = str.charAt(i) === '+' ? num + d & 4294967295 : num ^ d;
    }
    return num;
  }

  var num = key;
  for (var i = 0; i < str.length; i++) {
    num = RL(num + str.charCodeAt(i), '+-a^+6');
  }
  num = RL(num, '+-3^+b+-f');
  0 > num && (num = (num & 2147483647) + 2147483648);
  num %= 1E6;

  return num.toString() + '.' + (num ^ key);
}

module.exports = async function (word) {
  if (typeof word !== 'string' || word.length === 0) {
    throw new TypeError('word should be a string');
  }

  // replace '_' to ' ', and convert to lower case
  word = word.replace(/_/g, ' ').toLowerCase();

  var HOST = 'https://translate.google.com';
  var res = await fetch(HOST, {
    timeout: 10 * 1000
  });
  if (res.status !== 200) {
    var msg = util.format('request to %s failed, status code = %d (%s)', HOST, res.status, res.statusText);
    throw new Error(msg);
  }

  var $ = cheerio.load(await res.text());
  var scripts = $('#gt-c script').text();

  // get key
  var TKK, key = NaN;
  var code = scripts.match(/TKK=eval\(\'\(.*\)\'\);/gi)[0];
  eval(code);
  key = parseFloat(TKK);

  if (isNaN(key) === true) {
    throw new Error('key is not found');
  }

  var querystring = UrlFormat({
    query: {
      ie: 'UTF-8',
      q: word,    // encodeURIComponent
      tl: 'en',
      total: 1,
      idx: 0,
      textlen: word.length,
      tk: token(word, key),
      client: 't',
      prev: 'input',
      ttsspeed: 1   // slow = 0.24
    }
  });

  return HOST + '/translate_tts' + querystring;<commit_msg>[google] remove case-insensitive matching from regular expression<commit_after>var util      = require('util');
var UrlFormat = require('url').format;
var fetch     = require('node-fetch');
var cheerio   = require('cheerio');

function token (str, key) {
  function RL(num, str) {
    for (var i = 0; i < str.length - 2; i += 3) {
      var d = str.charAt(i + 2);
      d = d >= 'a' ? d.charCodeAt(0) - 87 : Number(d);
      d = str.charAt(i + 1) === '+' ? num >>> d : num << d;
      num = str.charAt(i) === '+' ? num + d & 4294967295 : num ^ d;
    }
    return num;
  }

  var num = key;
  for (var i = 0; i < str.length; i++) {
    num = RL(num + str.charCodeAt(i), '+-a^+6');
  }
  num = RL(num, '+-3^+b+-f');
  0 > num && (num = (num & 2147483647) + 2147483648);
  num %= 1E6;

  return num.toString() + '.' + (num ^ key);
}

module.exports = async function (word) {
  if (typeof word !== 'string' || word.length === 0) {
    throw new TypeError('word should be a string');
  }

  // replace '_' to ' ', and convert to lower case
  word = word.replace(/_/g, ' ').toLowerCase();

  var HOST = 'https://translate.google.com';
  var res = await fetch(HOST, {
    timeout: 10 * 1000
  });
  if (res.status !== 200) {
    var msg = util.format('request to %s failed, status code = %d (%s)', HOST, res.status, res.statusText);
    throw new Error(msg);
  }

  var $ = cheerio.load(await res.text());
  var scripts = $('#gt-c script').text();

  // get key
  var TKK, key = NaN;
  var code = scripts.match(/TKK=eval\(\'\(.*\)\'\);/g)[0];
  eval(code);
  key = parseFloat(TKK);

  if (isNaN(key) === true) {
    throw new Error('key is not found');
  }

  var querystring = UrlFormat({
    query: {
      ie: 'UTF-8',
      q: word,    // encodeURIComponent
      tl: 'en',
      total: 1,
      idx: 0,
      textlen: word.length,
      tk: token(word, key),
      client: 't',
      prev: 'input',
      ttsspeed: 1   // slow = 0.24
    }
  });

  return HOST + '/translate_tts' + querystring;
{'fraction_non_alphanumeric': 0.2, 'fraction_numerical': 0.031512071156289706, 'mean_word_length': 2.4526315789473685, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4495973', 'n_tokens_mistral': 1572, 'n_tokens_neox': 1461, 'n_words': 403}