<commit_before>/*eslint no-console:0 */
const request = require('request');
const mongoskin = require('mongoskin');
const db = mongoskin.db('mongodb://127.0.0.1:27017/throneteki');
const fs = require('fs');
const mkdirp = require('mkdirp');
const path = require('path');

var apiUrl = 'https://thronesdb.com/api/public/';

function fetchImage(urlPath, code, imagePath, timeout) {
    setTimeout(function() {
        console.log('Downloading image for ' + code);
        var url = 'https://thronesdb.com/' + urlPath;
        request(url).pipe(fs.createWriteStream(imagePath));
    }, timeout);
}

request.get(apiUrl + 'cards', function(error, res, body) {
    if(error) {
        console.error('Unable to fetch cards');
        return;
    }

    var cards = JSON.parse(body);

    var imageDir = path.join(__dirname, '..', 'public', 'img', 'cards');
    mkdirp(imageDir);

    var i = 0;

    cards.forEach(function(card) {
        var imagePath = path.join(imageDir, card.code + '.png');

        if(card.imagesrc && !fs.existsSync(imagePath)) {
            console.info(card.imagesrc, card.code, imagePath);
            fetchImage(card.imagesrc, card.code, imagePath, i++ * 200);
        }
    });

    db.collection('cards').remove({}, function() {
        db.collection('cards').insert(cards, function() {
            fs.writeFile('got-cards.json', JSON.stringify(cards), function() {
                console.info(cards.length + ' cards fetched');

                db.close();
            });
        });
    });
});

request.get(apiUrl + 'packs', function(error, res, body) {
    if(error) {
        console.error('Unable to fetch packs');
        return;<commit_msg>remove debugging no longer needed<commit_after>/*eslint no-console:0 */
const request = require('request');
const mongoskin = require('mongoskin');
const db = mongoskin.db('mongodb://127.0.0.1:27017/throneteki');
const fs = require('fs');
const mkdirp = require('mkdirp');
const path = require('path');

var apiUrl = 'https://thronesdb.com/api/public/';

function fetchImage(urlPath, code, imagePath, timeout) {
    setTimeout(function() {
        console.log('Downloading image for ' + code);
        var url = 'https://thronesdb.com/' + urlPath;
        request(url).pipe(fs.createWriteStream(imagePath));
    }, timeout);
}

request.get(apiUrl + 'cards', function(error, res, body) {
    if(error) {
        console.error('Unable to fetch cards');
        return;
    }

    var cards = JSON.parse(body);

    var imageDir = path.join(__dirname, '..', 'public', 'img', 'cards');
    mkdirp(imageDir);

    var i = 0;

    cards.forEach(function(card) {
        var imagePath = path.join(imageDir, card.code + '.png');

        if(card.imagesrc && !fs.existsSync(imagePath)) {
            fetchImage(card.imagesrc, card.code, imagePath, i++ * 200);
        }
    });

    db.collection('cards').remove({}, function() {
        db.collection('cards').insert(cards, function() {
            fs.writeFile('got-cards.json', JSON.stringify(cards), function() {
                console.info(cards.length + ' cards fetched');

                db.close();
            });
        });
    });
});

request.get(apiUrl + 'packs', function(error, res, body) {
    if(error) {
        console.error('Unable to fetch packs');
        return;
{'fraction_non_alphanumeric': 0.17267494698576188, 'fraction_numerical': 0.00969403211148137, 'mean_word_length': 2.8440046565774155, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2182269', 'n_tokens_mistral': 1062, 'n_tokens_neox': 1013, 'n_words': 202}