<commit_before>// ========================================================================
// Copyright (c) 2008-2009, Metaweb Technologies, Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
//       copyright notice, this list of conditions and the following
//       disclaimer in the documentation and/or other materials provided
//       with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
// ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
// TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
// DAMAGE.
// ========================================================================


/*
**  Rendering the spreadsheet back to the user
*/

function onDisplayOutputScreen() {
    addTimeout(() => checkLogin(),0);
    addTimeout(() => displayOutput(),0);
    addTimeout(() => prepareTriples(),0);
}
function onHideOutputScreen() {
    if (spreadsheetRendererYielder)
        spreadsheetRendererYielder.cancel();
    if (tripleGetterYielder)
        tripleGetterYielder.cancel();
    $("#outputData").val("");
}

/** @param {!Array.<(string|undefined)>} arr
  * @returns {!string}
  */
function encodeLine(arr) {
    var values = [];
    for(var i = 0; i < headerPaths.length; i++){
        var val = arr[i];
        if (typeof val === "undefined")
            values.push("");
        else if (!val.match(/(\t|\"|\n)/))
            values.push(arr[i])
        else {
            val = val.replace(/"/g, '""');
            values.push('"' + val + '"');
        }
    }
    return values.join("\t");
}

/**
  * @param {!tEntity} row
  * @return {!Array.<!string>}
  */
function encodeRow(row) {
    var lines = [[]];
    $.each(headerPaths, function(i, headerPath) {
        var val = row.get(headerPath, true);
        if ($.isArray(val)) {
            for (var j = 0; j < val.length; j++) {
                if (lines[j] == undefined) lines[j] = [];
                lines[j][i] = textValue(val[j]);
            }
        }
        else
            lines[0][i] = textValue(val);
    });
    return $.map(lines,encodeLine);
}

function displayOutput() {
    $("#outputData").val("One moment, rendering...");

    function setOutput(val) {
        $("#outputData").val(val);
    }

    if ($("input.outputFormat:checked").val() === "spreadsheet")
        renderSpreadsheet(setOutput)
    else
        renderJSON(setOutput);
}

function renderJSON(callback) {
    callback(JSON.stringify(rows, null, 2));
}

var spreadsheetRendererYielder;
function renderSpreadsheet(onComplete) {
    var lines = [];
    lines.push(encodeLine($.map(headerPaths, function(headerPath){return headerPath.toString()})));
    spreadsheetRendererYielder = new Yielder();
    politeEach(rows, function(idx, row) {
        lines = lines.concat(encodeRow(row));
    },
    function() {
        onComplete(lines.join("\n"));
    }, spreadsheetRendererYielder);
}


function prepareTriples(callback?:(triples:SuperFreeq.TripleLoadCommand[])=>any) {
    $(".renderingTriples").show();
    $(".triplesRendered").hide();
    var assertNaked = !! $("#assert_naked_properties").attr('checked');
    getSFTriples(entities, assertNaked,
      function(triples) {
        if (callback) {
          callback(triples);
        }

        politeMap(triples,function(val){return JSON.stringify(val)},
            function(encodedTriples:string[]) {
                var tripleString = encodedTriples.join("\n");
                $(".triplesDisplay").html(tripleString);
                $(".triple_count").html("" + encodedTriples.length);
                $('#payload').val(tripleString);
                $(".renderingTriples").hide();
                $(".triplesRendered").show();
            }
        );
    });
}



function getSFTriples(entities, assertNakedProperties,
                      callback:(triples:SuperFreeq.TripleLoadCommand[])=>any) {
  function tripToSfTrip(triple:OldFreeqTriple):SuperFreeq.TripleLoadCommand {

    if ($.type(triple.o) === "object") {
      var cvt_triples : SuperFreeq.CVTTriple[] = [];
      for (var prop in triple.o) {
        cvt_triples.push({
          pred: prop,
          obj: triple.o[prop]
        });
      }
      return {
        triple: {
          sub: triple.s,
          pred: triple.p,
        },
        cvt_triples: cvt_triples,
        assert_ids: true
      };
    }

    return {
      triple: {
        sub: triple.s,
        pred: triple.p,
        obj: triple.o
      },
      assert_ids: true
    };<commit_msg>convert objects to string before sending up to superfreeq<commit_after>// ========================================================================
// Copyright (c) 2008-2009, Metaweb Technologies, Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
//       copyright notice, this list of conditions and the following
//       disclaimer in the documentation and/or other materials provided
//       with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
// ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
// TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
// DAMAGE.
// ========================================================================


/*
**  Rendering the spreadsheet back to the user
*/

function onDisplayOutputScreen() {
    addTimeout(() => checkLogin(),0);
    addTimeout(() => displayOutput(),0);
    addTimeout(() => prepareTriples(),0);
}
function onHideOutputScreen() {
    if (spreadsheetRendererYielder)
        spreadsheetRendererYielder.cancel();
    if (tripleGetterYielder)
        tripleGetterYielder.cancel();
    $("#outputData").val("");
}

/** @param {!Array.<(string|undefined)>} arr
  * @returns {!string}
  */
function encodeLine(arr) {
    var values = [];
    for(var i = 0; i < headerPaths.length; i++){
        var val = arr[i];
        if (typeof val === "undefined")
            values.push("");
        else if (!val.match(/(\t|\"|\n)/))
            values.push(arr[i])
        else {
            val = val.replace(/"/g, '""');
            values.push('"' + val + '"');
        }
    }
    return values.join("\t");
}

/**
  * @param {!tEntity} row
  * @return {!Array.<!string>}
  */
function encodeRow(row) {
    var lines = [[]];
    $.each(headerPaths, function(i, headerPath) {
        var val = row.get(headerPath, true);
        if ($.isArray(val)) {
            for (var j = 0; j < val.length; j++) {
                if (lines[j] == undefined) lines[j] = [];
                lines[j][i] = textValue(val[j]);
            }
        }
        else
            lines[0][i] = textValue(val);
    });
    return $.map(lines,encodeLine);
}

function displayOutput() {
    $("#outputData").val("One moment, rendering...");

    function setOutput(val) {
        $("#outputData").val(val);
    }

    if ($("input.outputFormat:checked").val() === "spreadsheet")
        renderSpreadsheet(setOutput)
    else
        renderJSON(setOutput);
}

function renderJSON(callback) {
    callback(JSON.stringify(rows, null, 2));
}

var spreadsheetRendererYielder;
function renderSpreadsheet(onComplete) {
    var lines = [];
    lines.push(encodeLine($.map(headerPaths, function(headerPath){return headerPath.toString()})));
    spreadsheetRendererYielder = new Yielder();
    politeEach(rows, function(idx, row) {
        lines = lines.concat(encodeRow(row));
    },
    function() {
        onComplete(lines.join("\n"));
    }, spreadsheetRendererYielder);
}


function prepareTriples(callback?:(triples:SuperFreeq.TripleLoadCommand[])=>any) {
    $(".renderingTriples").show();
    $(".triplesRendered").hide();
    var assertNaked = !! $("#assert_naked_properties").attr('checked');
    getSFTriples(entities, assertNaked,
      function(triples) {
        if (callback) {
          callback(triples);
        }

        politeMap(triples,function(val){return JSON.stringify(val)},
            function(encodedTriples:string[]) {
                var tripleString = encodedTriples.join("\n");
                $(".triplesDisplay").html(tripleString);
                $(".triple_count").html("" + encodedTriples.length);
                $('#payload').val(tripleString);
                $(".renderingTriples").hide();
                $(".triplesRendered").show();
            }
        );
    });
}



function getSFTriples(entities, assertNakedProperties,
                      callback:(triples:SuperFreeq.TripleLoadCommand[])=>any) {
  function tripToSfTrip(triple:OldFreeqTriple):SuperFreeq.TripleLoadCommand {

    var obj = triple.o;

    if ($.type(obj) === "object") {
      var cvt_triples : SuperFreeq.CVTTriple[] = [];
      for (var prop in triple.o) {
        cvt_triples.push({
          pred: prop,
          obj: obj[prop]
        });
      }
      return {
        triple: {
          sub: triple.s,
          pred: triple.p,
        },
        cvt_triples: cvt_triples,
        assert_ids: true
      };
    }

    if ($.type(obj) === "number") {
      // SuperFreeq only wants strings, never numbers.
      // obj is the only place where this could have taken place.
      obj = '' + obj;
    }

    return {
      triple: {
        sub: triple.s,
        pred: triple.p,
        obj: obj
      },
      assert_ids: true
    };
{'fraction_non_alphanumeric': 0.16324139174328467, 'fraction_numerical': 0.0027041644131963224, 'mean_word_length': 2.6983333333333333, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4574127', 'n_tokens_mistral': 3256, 'n_tokens_neox': 3083, 'n_words': 815}