<commit_before>"use strict";

// Dependencies
var url = require("url");
var async = require("async");
var fs = require("fs");
var https = require("https");

// App variables
var BASE_URL = "https://en.wikipedia.org/wiki/";

var DOWNLOAD_DIR = "./test/spec/local_pages/";
var airlineFixtures = require("../fixtures/airline_routes.options.json");
var airportFixtures = require("../fixtures/airport_links.json");
var file_url = [
  "Category:Lists_of_airline_destinations"
];

for (var i = 0; i < airlineFixtures.length; i += 1) {
  file_url.push(airlineFixtures[i].name);
}

for (var j = 0; j < airportFixtures.length; j += 1) {
  file_url.push(airportFixtures[j].url);
}

var download_file_httpsGet = function (file_url, callback) {
  if (file_url.indexOf("https") === -1) {
    file_url = BASE_URL + file_url;
  }

  var file_name = decodeURI(url.parse(file_url).pathname.split("/").pop());

  file_name = file_name.split(":").pop();
  console.log("filename: " + file_name); //eslint-disable-line no-console
  var file = fs.createWriteStream(DOWNLOAD_DIR + file_name);

  https.get(file_url, function (res) {
    res.on("data", function (chunk) {
      file.write(chunk);
    }).on("error", function (err) {
      throw err;
    }).on("end", function () {
      file.end();
      console.log(file_name + " downloaded to " + DOWNLOAD_DIR); //eslint-disable-line no-console
      callback();
    });
  });

};

async.map(file_url, download_file_httpsGet, function (err) {
  if (err) {
    throw err;
  }
  console.log("\nLocal Pages Updated."); //eslint-disable-line no-console
});<commit_msg>adding iata list to local pages<commit_after>"use strict";

// Dependencies
var url = require("url");
var async = require("async");
var fs = require("fs");
var https = require("https");

// App variables
var BASE_URL = "https://en.wikipedia.org/wiki/";

var DOWNLOAD_DIR = "./test/spec/local_pages/";
var airlineFixtures = require("../fixtures/airline_routes.options.json");
var airportFixtures = require("../fixtures/airport_links.json");
var iataLocalList = require("../fixtures/airports_iata.data.json");
var file_url = [
  "Category:Lists_of_airline_destinations"
];

for (var i = 0; i < airlineFixtures.length; i += 1) {
  file_url.push(airlineFixtures[i].name);
}

for (var j = 0; j < airportFixtures.length; j += 1) {
  file_url.push(airportFixtures[j].url);
}

for (var k = 0; k < iataLocalList.length; k += 1) {
  file_url.push(iataLocalList[k]);
}

var download_file_httpsGet = function (file_url, callback) {
  if (file_url.indexOf("https") === -1) {
    file_url = BASE_URL + file_url;
  }

  var file_name = decodeURI(url.parse(file_url).pathname.split("/").pop());

  console.log("filename: " + file_name); //eslint-disable-line no-console
  var file = fs.createWriteStream(DOWNLOAD_DIR + file_name);

  https.get(file_url, function (res) {
    res.on("data", function (chunk) {
      file.write(chunk);
    }).on("error", function (err) {
      throw err;
    }).on("end", function () {
      file.end();
      console.log(file_name + " downloaded to " + DOWNLOAD_DIR);
      callback();
    });
  });

};

async.map(file_url, download_file_httpsGet, function (err) {
  if (err) {
    throw err;
  }
  console.log("\nLocal Pages Updated."); //eslint-disable-line no-console
});
{'fraction_non_alphanumeric': 0.17636308254645142, 'fraction_numerical': 0.003655193420651843, 'mean_word_length': 3.9383458646616543, 'pattern_counts': {'":': 1, '<': 8, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5426995', 'n_tokens_mistral': 1172, 'n_tokens_neox': 1141, 'n_words': 231}