<commit_before>var deckNotes;
var deckFields;
var deckName;
var ankiSeparator = '\x1f';

// Huge props to http://stackoverflow.com/a/9507713/500207
function tabulate(data, columns, containerString) {
    var table = d3.select(containerString).append("table"),
        thead = table.append("thead"),
        tbody = table.append("tbody");

    // append the header row
    thead.append("tr")
        .selectAll("th")
        .data(columns)
        .enter()
        .append("th")
            .text(function(column) { return column; });

    // create a row for each object in the data
    var rows = tbody.selectAll("tr")
        .data(data)
        .enter()
        .append("tr");

    // create a cell in each row for each column
    var cells = rows.selectAll("td")
        .data(function(row) {
            return columns.map(function(column) {
                return {column: column, value: row[column]};
            });
        })
        .enter()
        .append("td")
            .html(function(d) { return d.value; });

    return table;
}

function sqlToTable(uInt8ArraySQLdb) {
    var db = new SQL.Database(uInt8ArraySQLdb);

    col = db.exec("SELECT * FROM col");
    var modelsFunction = Function('return ' + col[0].values[0][9]);
    var models = modelsFunction();

    var fnames = [];
    for (key in models) {
        if (models.hasOwnProperty(key)) {
            // This should happen only once.
            deckName = models[key].name;
            models[key].flds.forEach(function (val, idx, arr) {
                fnames.push(val.name);
            });
        }
    }
    deckFields = fnames;

    // Notes table
    deckNotes = db.exec("SELECT * FROM notes");

    // Visualize!
    d3.select("body").append("div").attr("id", "anki").append("h1").text(deckName);
    //d3.select("#anki").append("div").text(deckFields);

    // Actual notes
    var notes = [];
    var arrayToObj = function(facts) {
        var myObj = {};
        for (var i = 0; i < facts.length; i++) {
            myObj[deckFields[i]] = facts[i];
        }
        return myObj;
    }
    deckNotes[0].values.forEach(function (val) {
        notes.push(arrayToObj(val[6].split(ankiSeparator)));
    });
    deckNotes = notes;
    tabulate(deckNotes, deckFields, "#anki");
}

function ankiBinaryToTable(ankiArray) {
    var compressed = new Uint8Array(ankiArray);
    var unzip = new Zlib.Unzip(compressed);
    var filenames = unzip.getFilenames();
    if (filenames.indexOf("collection.anki2") >= 0) {
        var plain = unzip.decompress("collection.anki2");
        sqlToTable(plain);
    }
}
function ankiURLToTable(ankiURL) {
    var zipxhr = new XMLHttpRequest();
    zipxhr.open('GET', ankiURL, true);
    zipxhr.responseType = 'arraybuffer';
    zipxhr.onload = function(e) {ankiBinaryToTable(this.response);}
    zipxhr.send();
}<commit_msg>Select only columns that are needed<commit_after>var deckNotes;
var deckFields;
var deckName;
var ankiSeparator = '\x1f';

// Huge props to http://stackoverflow.com/a/9507713/500207
function tabulate(data, columns, containerString) {
    var table = d3.select(containerString).append("table"),
        thead = table.append("thead"),
        tbody = table.append("tbody");

    // append the header row
    thead.append("tr")
        .selectAll("th")
        .data(columns)
        .enter()
        .append("th")
            .text(function(column) { return column; });

    // create a row for each object in the data
    var rows = tbody.selectAll("tr")
        .data(data)
        .enter()
        .append("tr");

    // create a cell in each row for each column
    var cells = rows.selectAll("td")
        .data(function(row) {
            return columns.map(function(column) {
                return {column: column, value: row[column]};
            });
        })
        .enter()
        .append("td")
            .html(function(d) { return d.value; });

    return table;
}

function sqlToTable(uInt8ArraySQLdb) {
    var db = new SQL.Database(uInt8ArraySQLdb);

    col = db.exec("SELECT models FROM col");
    var modelsFunction = Function('return ' + col[0].values[0][0]);
    var models = modelsFunction();

    var fnames = [];
    for (key in models) {
        if (models.hasOwnProperty(key)) {
            // This should happen only once.
            deckName = models[key].name;
            models[key].flds.forEach(function (val, idx, arr) {
                fnames.push(val.name);
            });
        }
    }
    deckFields = fnames;

    // Visualize!
    d3.select("#anki").append("h1").text(deckName);

    // Notes table
    deckNotes = db.exec("SELECT flds FROM notes");

    // Actual notes
    var notes = [];
    var arrayToObj = function(facts) {
        var myObj = {};
        for (var i = 0; i < facts.length; i++) {
            myObj[deckFields[i]] = facts[i];
        }
        return myObj;
    }
    deckNotes[0].values.forEach(function (val) {
        notes.push(arrayToObj(val[0].split(ankiSeparator)));
    });
    deckNotes = notes;
    tabulate(deckNotes, deckFields, "#anki");
}

function ankiBinaryToTable(ankiArray) {
    var compressed = new Uint8Array(ankiArray);
    var unzip = new Zlib.Unzip(compressed);
    var filenames = unzip.getFilenames();
    if (filenames.indexOf("collection.anki2") >= 0) {
        var plain = unzip.decompress("collection.anki2");
        sqlToTable(plain);
    }
}
function ankiURLToTable(ankiURL) {
    var zipxhr = new XMLHttpRequest();
    zipxhr.open('GET', ankiURL, true);
    zipxhr.responseType = 'arraybuffer';
    zipxhr.onload = function(e) {ankiBinaryToTable(this.response);}
    zipxhr.send();
}
{'fraction_non_alphanumeric': 0.14727305027535975, 'fraction_numerical': 0.010481435423698704, 'mean_word_length': 2.4141904184354153, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '687279', 'n_tokens_mistral': 1825, 'n_tokens_neox': 1765, 'n_words': 385}