<commit_before>const fs = require('fs');
const fsextra = require('fs-extra');
const path = require('path');
const util = require('util');
const logger = require('winston');

const sanitizeHtml = require('sanitize-html');

const request = require('request-promise');
const toml = require('toml');
const tomlify = require('tomlify-j0.4');
const blackfriday = require('./blackfriday.js');

const del = require('delete');
const exec = require('sync-exec');

const fsPathCode = './citra-games-wiki/games';
const fsPathWiki = './citra-games-wiki.wiki';
const fsPathHugoContent = '../../site/content/game';
const fsPathHugoBoxart = '../../site/static/images/game/boxart';
const fsPathHugoIcon = '../../site/static/images/game/icons';
const fsPathHugoScreenshots = '../../site/static/images/screenshots0';
const fsPathHugoSavefiles = '../../site/static/savefiles/';

function gitPull(directory, repository) {
  if (fs.existsSync(directory)) {
      logger.info(`Fetching latest from Github : ${directory}`);
      logger.info(`git --git-dir=${directory} pull`);
      exec(`git --git-dir=${directory} pull`);
  } else {
      logger.info(`Cloning repository from Github : ${directory}`);
      logger.info(`git clone ${repository}`);
      exec(`git clone ${repository}`);
  }
}

function getDirectories (srcpath) {
  return fs.readdirSync(srcpath)
    .filter(file => fs.lstatSync(path.join(srcpath, file)).isDirectory())
}

async function getGithubIssues() {
  var results = [];

  // Only loop through the first x pages to prevent API limiting.
  for (var page = 0; page <= 15; page++) {
    let options = {
      url: `https://api.github.com/repos/citra-emu/citra/issues?per_page=99&page=${page}`,
      headers: { 'User-Agent': 'citrabot' }
    };

    logger.verbose(`Requesting Github Issue Page ${page}.`);
    var body = await request.get(options);
    var obj = JSON.parse(body);

    if (obj == null || obj.length == 0) {
      logger.verbose(`No more Github issues found -- on page ${page}.`);
      break;
    } else {
      results = results.concat(obj);
    }
  }

  return results;
}

async function getTestcases() {
  let options = {
    url: 'https://api.citra-emu.org/telemetry/testcase/',
    headers: { 'User-Agent': 'citrabot' },
  };

  var body = await request.get(options);
  return JSON.parse(body).map(x => {
    return {
      title: x.title,
      compatibility: x.compatibility.toString(),
      date: x.date,
      version: x.version,
      buildName: x.buildName,
      buildDate: x.buildDate,
      cpu: x.cpu,
      gpu: x.gpu,
      os: x.os,
      author: x.author
    }
  });
}

// Fetch game information stored in Github repository.
gitPull('./citra-games-wiki', 'https://github.com/citra-emu/citra-games-wiki.git');

// Fetch game articles stored in Github wiki.
gitPull('./citra-games-wiki.wiki', 'https://github.com/citra-emu/citra-games-wiki.wiki.git');

// Fetch all issues from Github.
var githubIssues = [];
var wikiEntries = {};
var testcases = [];

async function setup() {
  //githubIssues = await getGithubIssues();
  //logger.info(`Imported ${githubIssues.length} issues from Github.`);
  
  testcases = await getTestcases();
  logger.info(`Obtained ${testcases.length} testcases from Telemetry API.`);
};

setup().then(function() {
  // Make sure the output directories in Hugo exist.
  [fsPathHugoContent, fsPathHugoBoxart, fsPathHugoIcon, fsPathHugoSavefiles, fsPathHugoScreenshots].forEach(function (path) {
    if (fs.existsSync(path) == false) {
        logger.info(`Creating Hugo output directory: ${path}`);
        fs.mkdirSync(path);
    }
  });
})
.then(function() {<commit_msg>Removed comments for debugging.<commit_after>const fs = require('fs');
const fsextra = require('fs-extra');
const path = require('path');
const util = require('util');
const logger = require('winston');

const sanitizeHtml = require('sanitize-html');

const request = require('request-promise');
const toml = require('toml');
const tomlify = require('tomlify-j0.4');
const blackfriday = require('./blackfriday.js');

const del = require('delete');
const exec = require('sync-exec');

const fsPathCode = './citra-games-wiki/games';
const fsPathWiki = './citra-games-wiki.wiki';
const fsPathHugoContent = '../../site/content/game';
const fsPathHugoBoxart = '../../site/static/images/game/boxart';
const fsPathHugoIcon = '../../site/static/images/game/icons';
const fsPathHugoScreenshots = '../../site/static/images/screenshots0';
const fsPathHugoSavefiles = '../../site/static/savefiles/';

function gitPull(directory, repository) {
  if (fs.existsSync(directory)) {
      logger.info(`Fetching latest from Github : ${directory}`);
      logger.info(`git --git-dir=${directory} pull`);
      exec(`git --git-dir=${directory} pull`);
  } else {
      logger.info(`Cloning repository from Github : ${directory}`);
      logger.info(`git clone ${repository}`);
      exec(`git clone ${repository}`);
  }
}

function getDirectories (srcpath) {
  return fs.readdirSync(srcpath)
    .filter(file => fs.lstatSync(path.join(srcpath, file)).isDirectory())
}

async function getGithubIssues() {
  var results = [];

  // Only loop through the first x pages to prevent API limiting.
  for (var page = 0; page <= 15; page++) {
    let options = {
      url: `https://api.github.com/repos/citra-emu/citra/issues?per_page=99&page=${page}`,
      headers: { 'User-Agent': 'citrabot' }
    };

    logger.verbose(`Requesting Github Issue Page ${page}.`);
    var body = await request.get(options);
    var obj = JSON.parse(body);

    if (obj == null || obj.length == 0) {
      logger.verbose(`No more Github issues found -- on page ${page}.`);
      break;
    } else {
      results = results.concat(obj);
    }
  }

  return results;
}

async function getTestcases() {
  let options = {
    url: 'https://api.citra-emu.org/telemetry/testcase/',
    headers: { 'User-Agent': 'citrabot' },
  };

  var body = await request.get(options);
  return JSON.parse(body).map(x => {
    return {
      title: x.title,
      compatibility: x.compatibility.toString(),
      date: x.date,
      version: x.version,
      buildName: x.buildName,
      buildDate: x.buildDate,
      cpu: x.cpu,
      gpu: x.gpu,
      os: x.os,
      author: x.author
    }
  });
}

// Fetch game information stored in Github repository.
gitPull('./citra-games-wiki', 'https://github.com/citra-emu/citra-games-wiki.git');

// Fetch game articles stored in Github wiki.
gitPull('./citra-games-wiki.wiki', 'https://github.com/citra-emu/citra-games-wiki.wiki.git');

// Fetch all issues from Github.
var githubIssues = [];
var wikiEntries = {};
var testcases = [];

async function setup() {
  githubIssues = await getGithubIssues();
  logger.info(`Imported ${githubIssues.length} issues from Github.`);
  
  testcases = await getTestcases();
  logger.info(`Obtained ${testcases.length} testcases from Telemetry API.`);
};

setup().then(function() {
  // Make sure the output directories in Hugo exist.
  [fsPathHugoContent, fsPathHugoBoxart, fsPathHugoIcon, fsPathHugoSavefiles, fsPathHugoScreenshots].forEach(function (path) {
    if (fs.existsSync(path) == false) {
        logger.info(`Creating Hugo output directory: ${path}`);
        fs.mkdirSync(path);
    }
  });
})
.then(function() {
{'fraction_non_alphanumeric': 0.16863661352391424, 'fraction_numerical': 0.002473886750962067, 'mean_word_length': 3.533956386292835, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7446414', 'n_tokens_mistral': 2401, 'n_tokens_neox': 2350, 'n_words': 533}