<commit_before>var fs = require('fs');
var XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
var url = 'http://pokedata.c4e3f8c7.svc.dockerapp.io:65014/api/pokemon/sighting/';

var destination = 'arff/apiData.arff';

httpGetAsync(url, destination);

function httpGetAsync(url, destination) {
    var xmlHttp = new XMLHttpRequest();
    xmlHttp.onreadystatechange = function () {
        if (xmlHttp.readyState == 4 && xmlHttp.status == 200) {
            var apiData = JSON.parse(xmlHttp.responseText);
            jsonToArff(apiData, 'apiData', destination);
        }
    };
    xmlHttp.open("GET", url, true);
    xmlHttp.send();
    return xmlHttp.responseText;
}

function jsonToArff(json_data, fileName, destination) {
    var valid_data = removeIncompleteData(json_data);
    var arff = '@RELATION ' + fileName + '\n\n';

    // attributes
    arff += '@ATTRIBUTE _id string\n';
    arff += '@ATTRIBUTE source string\n';
    arff += '@ATTRIBUTE latitude numeric\n';
    arff += '@ATTRIBUTE longitude numeric\n';
    arff += '@ATTRIBUTE appearedOn numeric\n';
    arff += '@ATTRIBUTE __v numeric\n';

    // class
    var classKey = 'pokemonId';
    var classLabels = allValuesForKeyInData(classKey, valid_data);
    arff += '@ATTRIBUTE class {' + classLabels.join(', ') + '}\n\n';

    // data
    arff += '@DATA\n';
    valid_data.forEach(function (element) {
        arff += element["_id"].replace(/\s+/g, '') + ',';
        arff += element["source"].replace(/\s+/g, '') + ',';
        arff += element["location"]["coordinates"][0] + ',';
        arff += element["location"]["coordinates"][1] + ',';
        arff += Date.parse(element["appearedOn"]) + ',';
        arff += element["__v"] + ',';
        arff += element[classKey] + '\n';
    });

    fs.writeFileSync(destination, arff, 'utf8');
}

/**
 * remove data entries which do not provided all required features
 * @param json_data data to filter
 * @returns {Array} array which contains only entries which provide all necessary features
 */
function removeIncompleteData(json_data) {
    var complete_data = [];

    json_data.forEach(function (element) {
        if (element["location"] !== null) {
            complete_data.push(element);
        }
    });

    return complete_data;
}

function allValuesForKeyInData(key, json_data) {
    var values = [];

    json_data.forEach(function (row) {
        values.push(row[key]);
    });

    return Array.from(new Set(values));
}<commit_msg>moved .arff creation code into the dataSet_creator.js file and use it from here to allow support of different feature sources<commit_after>var fs = require('fs');
var DS = require('./dataSet_creator.js').DC;
var XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
var url = 'http://pokedata.c4e3f8c7.svc.dockerapp.io:65014/api/pokemon/sighting/';
var destination = 'arff/apiDataExtended.arff';

httpGetAsync(url, destination);

function httpGetAsync(url, destination) {
    var xmlHttp = new XMLHttpRequest();
    xmlHttp.onreadystatechange = function () {
        if (xmlHttp.readyState == 4 && xmlHttp.status == 200) {
            var apiData = JSON.parse(xmlHttp.responseText);
            DS.storeArffFile("feature_config.json", apiData.data, destination);
        }
    };
    xmlHttp.open("GET", url, true);
    xmlHttp.send();
    return xmlHttp.responseText;
}
{'fraction_non_alphanumeric': 0.15345498055638648, 'fraction_numerical': 0.008674842955429255, 'mean_word_length': 3.4057971014492754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6907423', 'n_tokens_mistral': 1077, 'n_tokens_neox': 1057, 'n_words': 225}