<commit_before>import Champion from "./Champion";
import {highscores} from "./server";
import Config from "./Config";
import fs = require("fs");
import http = require("http");
import https = require("https");
import path = require("path");
import stream = require("stream");
import VError = require("verror");
import mkdirp = require("mkdirp");
const gunzip = require("gunzip-maybe");
const tar = require("tar-stream");

/** The directory where public images are stored */
export const imagesPath: string = path.join(Config.staticDataPath, "icons");
/** The directory where champion icons are stored */
const championIconsPath: string = path.join(imagesPath, "championIcons");
/** The directory where profile icons are stored */
const profileIconsPath: string = path.join(imagesPath, "profileIcons");
/** The file where the champion list is saved */
const championListPath: string = path.join(Config.staticDataPath, "champion.json");
/** The path of a text file containing the DDragon version of the currently downloaded data */
const ddragonVersionPath: string = path.join(Config.staticDataPath, "ddragonVersion.txt");
/** The DDragon version of the currently downloaded static data */
let currentDDragonVersion: string;

/**
 * Updates the current DDragon version, champion list, and icons to the latest version.
 * Does nothing if everything is already up to date.
 * @async
 * @throws {Error} Thrown if an error occurs while updating data. If this happens, some static data may be updated and
 * other parts will continue using old data (if an older version of the data was already downloaded).
 */
export const updateStaticData = async (): Promise<void> => {
	return new Promise<void>(async (resolve: Function, reject: Function) => {
		console.log("Updating static data...");
		// Create the static data directory (if it doesn't already exist)
		try {
			if (!fs.existsSync(Config.staticDataPath)) {
				mkdirp.sync(Config.staticDataPath);
				console.log("Created static data directory");
			}
		} catch (ex) {
			reject(new VError(ex, `Unable to create static data directory ${Config.staticDataPath}`));
			return;
		}
		// Determine the DDragon version of the currently downloaded static data (if it hasn't already been determined)
		if (!currentDDragonVersion) {
			try {
				if (fs.existsSync(ddragonVersionPath)) {
					currentDDragonVersion = fs.readFileSync(ddragonVersionPath, "utf8");
				}
			} catch (ex) {
				// This error isn't critical (static data will just be redownloaded, even if it isn't necessary), so it's just logged instead of thrown
				console.error(VError.fullStack(new VError(ex, `Error loading DDragon version from ${ddragonVersionPath}`)));
			}
		}
		/** The latest published DDragon version */
		let latestVersion: string;
		try {
			latestVersion = await getLatestDDragonVersion();
		} catch (ex) {
			// Continue using the currently downloaded version (if it exists) if there is an error getting the latest version
			if (currentDDragonVersion) {
				latestVersion = currentDDragonVersion;
				console.error(VError.fullStack(new VError(ex, "Error getting latest DDragon version, will continue to user currently downloaded version")));
			} else {
				reject(new VError(ex, "Error getting latest DDragon version"));
				return;
			}
		}

		if (!currentDDragonVersion || latestVersion !== currentDDragonVersion) {
			try {
				await downloadData(latestVersion);
				currentDDragonVersion = latestVersion;
				// Save the latest DDragon version to ddragonVersion.txt
				try {
					fs.writeFileSync(ddragonVersionPath, latestVersion, "utf8");
				} catch (ex) {
					// This error isn't critical, so it's just logged instead of thrown
					console.error(VError.fullStack(new VError(ex, "Error saving latest DDragon version")));
				}

				console.log("Updated static data. Latest version: " + currentDDragonVersion);
				resolve();
			} catch (ex) {
				reject(new VError(ex, "Error updating static data"));
			}
		} else {
			// Champions still need to be loaded when the server starts
			if (!Champion.CHAMPIONS) {
				updateChampions();
			}
			console.log("Static data is already up to date");
			resolve();
		}
	});
};

/**
 * Downloads the latest dragontail archive and extracts profile icons, champion icons, and the champion list
 * @param ddragonVersion The latest DDragon version (e.g. "7.14.1")
 * @async
 * @throws {Error} Thrown if an error occurs while downloading or extracting static data
 */
const downloadData = (ddragonVersion: string): Promise<void> => {
	return new Promise<void>((resolve: Function, reject: Function) => {
		console.log("Downloading static data...");
		try {
			if (!fs.existsSync(championIconsPath)) {
				mkdirp.sync(championIconsPath);
			}
			if (!fs.existsSync(profileIconsPath)) {
				mkdirp.sync(profileIconsPath);
			}
		} catch (ex) {
			reject(new VError(ex, "Unable to create icon directories"));
			return;
		}

		const url: string = `https://ddragon.leagueoflegends.com/cdn/dragontail-${ddragonVersion}.tgz`;
		https.get(url, (response: http.IncomingMessage) => {
			if (response.statusCode === 200) {
				/** A promise for each file that needs to be saved. Each promise will be resolved when the file has been saved, or rejected if an error occurs */
				const promises: Promise<void>[] = [];

				const tarStream = tar.extract();
				tarStream.on("error", (err: Error) => {
					reject(new VError(err, "Error reading tarball stream"));
				});
				let entriesChecked: number = 0;
				tarStream.on("entry", (header: {name: string}, entryStream: stream.Readable, next: Function) => {
					if (++entriesChecked % 1000 === 0) {
						console.log(`Checked ${entriesChecked} entries in the tarball...`);
					}
					if (header.name.indexOf(`./${ddragonVersion}/img/profileicon/`) === 0 && !header.name.endsWith("/")) {
						const promise: Promise<void> = saveEntry(entryStream, profileIconsPath, header.name);
						// This is needed to suppress an UnhandledPromiseRejectionWarning (the rejection will actually be handled later by Promise.all())
						promise.catch(() => {});
						promises.push(promise);
					} else if (header.name.indexOf(`./${ddragonVersion}/img/champion/`) === 0 && !header.name.endsWith("/")) {
						const promise: Promise<void> = saveEntry(entryStream, championIconsPath, header.name);
						// This is needed to suppress an UnhandledPromiseRejectionWarning (the rejection will actually be handled later by Promise.all())
						promise.catch(() => {});
						promises.push(promise);
					} else if (header.name === `./${ddragonVersion}/data/en_US/champion.json`) {
						const promise: Promise<void> = saveEntry(entryStream, Config.staticDataPath, header.name);
						promise.then(() => {
							updateChampions();
							// This is needed to suppress an UnhandledPromiseRejectionWarning (the rejection will actually be handled later by Promise.all())
						}, () => {});
						promises.push(promise);
					} else {
						/* "The tar archive is streamed sequentially, meaning you must drain each entry's stream as
						you get them or else the main extract stream will receive backpressure and stop reading."
						- https://github.com/mafintosh/tar-stream/blob/master/README.md */
						entryStream.resume();
					}
					next();
				});
				tarStream.on("finish", () => {
					console.log("Finished checking tarball, waiting for files to finish saving...");
					Promise.all(promises).then(() => {
						console.log("All files finished saving");
						resolve();
					}, (err) => {
						reject(new VError(err, "Error updating static data"));
					});<commit_msg>Fix staticDataUpdater.ts<commit_after>import Champion from "./Champion";
import {highscores} from "./server";
import Config from "./Config";
import fs = require("fs");
import http = require("http");
import https = require("https");
import path = require("path");
import stream = require("stream");
import VError = require("verror");
import mkdirp = require("mkdirp");
import XRegExp = require("xregexp");

const gunzip = require("gunzip-maybe");
const tar = require("tar-stream");

/** The directory where public images are stored */
export const imagesPath: string = path.join(Config.staticDataPath, "icons");
/** The directory where champion icons are stored */
const championIconsPath: string = path.join(imagesPath, "championIcons");
/** The directory where profile icons are stored */
const profileIconsPath: string = path.join(imagesPath, "profileIcons");
/** The file where the champion list is saved */
const championListPath: string = path.join(Config.staticDataPath, "champion.json");
/** The path of a text file containing the DDragon version of the currently downloaded data */
const ddragonVersionPath: string = path.join(Config.staticDataPath, "ddragonVersion.txt");
/** The DDragon version of the currently downloaded static data */
let currentDDragonVersion: string;

/**
 * Updates the current DDragon version, champion list, and icons to the latest version.
 * Does nothing if everything is already up to date.
 * @async
 * @throws {Error} Thrown if an error occurs while updating data. If this happens, some static data may be updated and
 * other parts will continue using old data (if an older version of the data was already downloaded).
 */
export const updateStaticData = async (): Promise<void> => {
	return new Promise<void>(async (resolve: Function, reject: Function) => {
		console.log("Updating static data...");
		// Create the static data directory (if it doesn't already exist)
		try {
			if (!fs.existsSync(Config.staticDataPath)) {
				mkdirp.sync(Config.staticDataPath);
				console.log("Created static data directory");
			}
		} catch (ex) {
			reject(new VError(ex, `Unable to create static data directory ${Config.staticDataPath}`));
			return;
		}
		// Determine the DDragon version of the currently downloaded static data (if it hasn't already been determined)
		if (!currentDDragonVersion) {
			try {
				if (fs.existsSync(ddragonVersionPath)) {
					currentDDragonVersion = fs.readFileSync(ddragonVersionPath, "utf8");
				}
			} catch (ex) {
				// This error isn't critical (static data will just be redownloaded, even if it isn't necessary), so it's just logged instead of thrown
				console.error(VError.fullStack(new VError(ex, `Error loading DDragon version from ${ddragonVersionPath}`)));
			}
		}
		/** The latest published DDragon version */
		let latestVersion: string;
		try {
			latestVersion = await getLatestDDragonVersion();
		} catch (ex) {
			// Continue using the currently downloaded version (if it exists) if there is an error getting the latest version
			if (currentDDragonVersion) {
				latestVersion = currentDDragonVersion;
				console.error(VError.fullStack(new VError(ex, "Error getting latest DDragon version, will continue to user currently downloaded version")));
			} else {
				reject(new VError(ex, "Error getting latest DDragon version"));
				return;
			}
		}

		if (!currentDDragonVersion || latestVersion !== currentDDragonVersion) {
			try {
				await downloadData(latestVersion);
				currentDDragonVersion = latestVersion;
				// Save the latest DDragon version to ddragonVersion.txt
				try {
					fs.writeFileSync(ddragonVersionPath, latestVersion, "utf8");
				} catch (ex) {
					// This error isn't critical, so it's just logged instead of thrown
					console.error(VError.fullStack(new VError(ex, "Error saving latest DDragon version")));
				}

				console.log("Updated static data. Latest version: " + currentDDragonVersion);
				resolve();
			} catch (ex) {
				reject(new VError(ex, "Error updating static data"));
			}
		} else {
			// Champions still need to be loaded when the server starts
			if (!Champion.CHAMPIONS) {
				updateChampions();
			}
			console.log("Static data is already up to date");
			resolve();
		}
	});
};

/**
 * Downloads the latest dragontail archive and extracts profile icons, champion icons, and the champion list
 * @param ddragonVersion The latest DDragon version (e.g. "7.14.1")
 * @async
 * @throws {Error} Thrown if an error occurs while downloading or extracting static data
 */
const downloadData = (ddragonVersion: string): Promise<void> => {
	return new Promise<void>((resolve: Function, reject: Function) => {
		console.log("Downloading static data...");
		try {
			if (!fs.existsSync(championIconsPath)) {
				mkdirp.sync(championIconsPath);
			}
			if (!fs.existsSync(profileIconsPath)) {
				mkdirp.sync(profileIconsPath);
			}
		} catch (ex) {
			reject(new VError(ex, "Unable to create icon directories"));
			return;
		}

		const url: string = `https://ddragon.leagueoflegends.com/cdn/dragontail-${ddragonVersion}.tgz`;
		https.get(url, (response: http.IncomingMessage) => {
			if (response.statusCode === 200) {
				/** A promise for each file that needs to be saved. Each promise will be resolved when the file has been saved, or rejected if an error occurs */
				const promises: Promise<void>[] = [];

				const tarStream = tar.extract();
				tarStream.on("error", (err: Error) => {
					reject(new VError(err, "Error reading tarball stream"));
				});

				const championJsonRegex = XRegExp(`^(.\\/)?${XRegExp.escape(ddragonVersion)}\\/data\\/en_US\\/champion\\.json$`);
				const profileIconRegex = XRegExp(`^(.\\/)?${XRegExp.escape(ddragonVersion)}\\/img\\/profileicon\\/.+[^\\/]$`);
				const championIconRegex = XRegExp(`^(.\\/)?${XRegExp.escape(ddragonVersion)}\\/img\\/champion\\/.+[^\\/]$`);

				let entriesChecked: number = 0;
				tarStream.on("entry", (header: { name: string }, entryStream: stream.Readable, next: Function) => {
					if (++entriesChecked % 1000 === 0) {
						console.log(`Checked ${entriesChecked} entries in the tarball...`);
					}
					if (profileIconRegex.test(header.name)) {
						const promise: Promise<void> = saveEntry(entryStream, profileIconsPath, header.name);
						// This is needed to suppress an UnhandledPromiseRejectionWarning (the rejection will actually be handled later by Promise.all())
						promise.catch(() => {});
						promises.push(promise);
					} else if (championIconRegex.test(header.name)) {
						const promise: Promise<void> = saveEntry(entryStream, championIconsPath, header.name);
						// This is needed to suppress an UnhandledPromiseRejectionWarning (the rejection will actually be handled later by Promise.all())
						promise.catch(() => { });
						promises.push(promise);
					} else if (championJsonRegex.test(header.name)) {
						const promise: Promise<void> = saveEntry(entryStream, Config.staticDataPath, header.name);
						promise.then(() => {
							updateChampions();
							// This is needed to suppress an UnhandledPromiseRejectionWarning (the rejection will actually be handled later by Promise.all())
						}, () => { });
						promises.push(promise);
					} else {
						/* "The tar archive is streamed sequentially, meaning you must drain each entry's stream as
						you get them or else the main extract stream will receive backpressure and stop reading."
						- https://github.com/mafintosh/tar-stream/blob/master/README.md */
						entryStream.resume();
					}
					next();
				});
				tarStream.on("finish", () => {
					console.log(`Finished checking tarball, waiting for ${promises.length} files to finish saving...`);
					Promise.all(promises).then(() => {
						console.log("All files finished saving");
						resolve();
					}, (err) => {
						reject(new VError(err, "Error updating static data"));
					});
{'fraction_non_alphanumeric': 0.12737408439748493, 'fraction_numerical': 0.0020742853438776173, 'mean_word_length': 3.891566265060241, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 47, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6166084', 'n_tokens_mistral': 5147, 'n_tokens_neox': 4361, 'n_words': 1413}