<commit_before>#!/usr/bin/env node
/*
 * scrape_champ_ids.js -- scrape Mobafire for champion name -> ID mapping
 *
 * Copyright (C) 2017 <NAME>
 *
 * This software may be modified and distributed under the terms
 * of the MIT license. See the LICENSE file for details.
 */

const http = require("http");

http.get("http://www.mobafire.com/league-of-legends/champions", (response) => {
  const { statusCode } = response;
  const contentType = response.headers["content-type"];

  if (statusCode !== 200) {
    throw new Error(`HTTP request failed with status ${statusCode}`);
  } else if (!/text\/html/.test(contentType)) {
    throw new Error(`HTTP request failed with invalid Content-Type ${contentType}`);
  }

  let html = "";

  response.setEncoding("utf8");

  response.on("data", (chunk) => { html += chunk; });

  response.on("end", () => {
    const champions = {};<commit_msg>Mobafire uses https: now :)<commit_after>#!/usr/bin/env node
/*
 * scrape_champ_ids.js -- scrape Mobafire for champion name -> ID mapping
 *
 * Copyright (C) 2017 <NAME>
 *
 * This software may be modified and distributed under the terms
 * of the MIT license. See the LICENSE file for details.
 */

const https = require("https");

https.get("https://www.mobafire.com/league-of-legends/champions", (response) => {
  const { statusCode } = response;
  const contentType = response.headers["content-type"];

  if (statusCode !== 200) {
    throw new Error(`HTTP request failed with status ${statusCode}`);
  } else if (!/text\/html/.test(contentType)) {
    throw new Error(`HTTP request failed with invalid Content-Type ${contentType}`);
  }

  let html = "";

  response.setEncoding("utf8");

  response.on("data", (chunk) => { html += chunk; });

  response.on("end", () => {
    const champions = {};
{'fraction_non_alphanumeric': 0.16051454138702462, 'fraction_numerical': 0.008948545861297539, 'mean_word_length': 3.7706666666666666, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6245747', 'n_tokens_mistral': 565, 'n_tokens_neox': 534, 'n_words': 160}