<commit_before>import querystring from 'querystring';
import request from 'request-promise-native';

import WarcraftLogsApiError from './WarcraftLogsApiError';

const WCL_DOMAIN = process.env.WARCRAFT_LOGS_DOMAIN;
const WCL_MAINTENANCE_STRING = 'Warcraft Logs is down for maintenance';
export const WCL_REPORT_DOES_NOT_EXIST_HTTP_CODE = 400;
const USER_AGENT = process.env.USER_AGENT;
const WCL_API_KEY = process.env.WCL_API_KEY;

function getWclApiUrl(path, query) {
  return `${WCL_DOMAIN}/v1/${path}?${querystring.stringify({
    api_key: WCL_API_KEY,
    ...query,
  })}`;
}
function tryJsonParse(string) {
  let json = null;
  try {
    json = JSON.parse(string);
  } catch (jsonErr) {
    // ignore
  }
  return json;
}
export default async function fetchFromWarcraftLogsApi(path, query) {
  const url = getWclApiUrl(path, query);
  try {
    const jsonString = await request.get({
      url,
      headers: {
        'User-Agent': USER_AGENT,
      },
      // using gzip is 80% quicker
      gzip: true,
      // we'll be making several requests, so pool connections
      forever: true,
    });

    // WCL maintenance mode returns 200 http code :(
    if (jsonString.includes(WCL_MAINTENANCE_STRING)) {
      throw new WarcraftLogsApiError(503, WCL_MAINTENANCE_STRING);
    }
    // WCL has a tendency to throw non-JSON errors with a 200 HTTP exception, this ensures they're not accepted and cached.
    // Decoding JSON takes a long time, grabbing the first character is near instant and has high accuracy.
    const firstCharacter = jsonString.substr(0, 1);
    if (firstCharacter !== '{' && firstCharacter !== '[') {
      throw new WarcraftLogsApiError(500, 'Corrupt Warcraft Logs API response received', jsonString);
    }

    return jsonString;
  } catch (err) {
    if (err instanceof WarcraftLogsApiError) {
      // Already the correct format, pass it along
      throw err;
    }
    if (err.error) {
      const json = tryJsonParse(err.error);
      if (json) {
        throw new WarcraftLogsApiError(err.statusCode, json.error);
      }
    }

    const message = err.error || err.message;
    throw new WarcraftLogsApiError(err.statusCode, message);
  }
}<commit_msg>server: Abort requests after 5 sec<commit_after>import querystring from 'querystring';
import request from 'request-promise-native';

import WarcraftLogsApiError from './WarcraftLogsApiError';

const WCL_DOMAIN = process.env.WARCRAFT_LOGS_DOMAIN;
const WCL_MAINTENANCE_STRING = 'Warcraft Logs is down for maintenance';
export const WCL_REPORT_DOES_NOT_EXIST_HTTP_CODE = 400;
const USER_AGENT = process.env.USER_AGENT;
const WCL_API_KEY = process.env.WCL_API_KEY;
const TIMEOUT = 5000; // ms after which to abort the request

function getWclApiUrl(path, query) {
  return `${WCL_DOMAIN}/v1/${path}?${querystring.stringify({
    api_key: WCL_API_KEY,
    ...query,
  })}`;
}
function tryJsonParse(string) {
  let json = null;
  try {
    json = JSON.parse(string);
  } catch (jsonErr) {
    // ignore
  }
  return json;
}
export default async function fetchFromWarcraftLogsApi(path, query) {
  const url = getWclApiUrl(path, query);
  try {
    const jsonString = await request.get({
      url,
      headers: {
        'User-Agent': USER_AGENT,
      },
      // using gzip is 80% quicker
      gzip: true,
      // we'll be making several requests, so pool connections
      forever: true,
      timeout: TIMEOUT,
    });

    // WCL maintenance mode returns 200 http code :(
    if (jsonString.includes(WCL_MAINTENANCE_STRING)) {
      throw new WarcraftLogsApiError(503, WCL_MAINTENANCE_STRING);
    }
    // WCL has a tendency to throw non-JSON errors with a 200 HTTP exception, this ensures they're not accepted and cached.
    // Decoding JSON takes a long time, grabbing the first character is near instant and has high accuracy.
    const firstCharacter = jsonString.substr(0, 1);
    if (firstCharacter !== '{' && firstCharacter !== '[') {
      throw new WarcraftLogsApiError(500, 'Corrupt Warcraft Logs API response received', jsonString);
    }

    return jsonString;
  } catch (err) {
    if (err instanceof WarcraftLogsApiError) {
      // Already the correct format, pass it along
      throw err;
    }
    if (err.error) {
      if (err.error.code === 'ETIMEDOUT') {
        throw new WarcraftLogsApiError(504, 'Warcraft Logs took too long to respond.');
      }
      const json = tryJsonParse(err.error);
      if (json) {
        throw new WarcraftLogsApiError(err.statusCode, json.error);
      }
    }

    const message = err.error || err.message;
    throw new WarcraftLogsApiError(err.statusCode, message);
  }
}
{'fraction_non_alphanumeric': 0.1026361279170268, 'fraction_numerical': 0.010371650821089023, 'mean_word_length': 3.314072693383038, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1678980', 'n_tokens_mistral': 1515, 'n_tokens_neox': 1479, 'n_words': 416}