<commit_before>"use strict"

const Hash = require('crypto').createHash;

// RDF constants
const RDF_SYNTAX = '<http://www.w3.org/1999/02/22-rdf-syntax-ns#';
const FLOW_VOCAB = '<http://schema.jillix.net/vocab/';
const SCHEMA = '<http://schema.org/';

function UID (len) {
    len = len || 23;
    let i = 0, random = '';
    for (; i < len; ++i) {
        random += '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'[0 | Math.random() * 62];
    }
    return '_:' + Hash('md5').update(random).digest('hex');
}

function inArray(array, id) {
    for (let i = 0, l = array.length; i < l; ++i) {
        if (array[i].id === id) {
            return true;
        }
    }
}

function nodesExists (client, nodes, callback) {
    client.g.V.apply(client.g, nodes).GetLimit(nodes.length, (err, result) => {

        if (err || (result && result.error)) {
            return callback(err || result.error);
        } 

        const exists = [];
        const inexistent = [];

        if (!result) {
            return callback(null, exists, nodes);
        }

        nodes.forEach(node => {
            if (inArray(result, node)) {
                exists.push(node);
            } else {
                inexistent.push(node);
            }
        });

        callback(null, exists, inexistent);
    });
}

exports.createNodeOfType = (client, type) => {

    const node = UID();

    switch (type) {
        case "onError":
        case "onEnd":
            type = 'sequence';
            break;
        case "next":
            // TODO handler type not in data
            // TODO add seq > handler > data/emit/stream
            type = 'data';
        break;
    }

    type = type.charAt(0).toUpperCase() + type.slice(1);

    return {
        uid: node,
        triples: [{
            subject: node,
            predicate: RDF_SYNTAX + 'type>',
            object: FLOW_VOCAB + type + '>'
        }]
    };
};

exports.ensureHashs = (client, values, callback) => {

    const hashs = [];
    values.forEach((value, index) => {
        hashs.push('_:' + Hash('md5').update(value).digest('hex'));
    });

    nodesExists(client, hashs, (err, exists, inexistent) => {

        if (err) {
            return callback(err);
        }

        if (inexistent.length === 0) {
            return callback(null, hashs);
        }

        const triples = [];
        inexistent.forEach((hash) => {
            triples.push({
                subject: hash,
                predicate: RDF_SYNTAX + 'string>',
                object: values[hashs.indexOf(hash)]
            });
        });

        callback(null, hashs, triples);
    });
};

exports.removeOut = (client, node, edges) => {
    console.log('Network: remove "' + edges + '" of node', node);
};<commit_msg>delete first level of outgoing edges<commit_after>"use strict"

const Hash = require('crypto').createHash;

// RDF constants
const RDF_SYNTAX = '<http://www.w3.org/1999/02/22-rdf-syntax-ns#';
const FLOW_VOCAB = '<http://schema.jillix.net/vocab/';
const SCHEMA = '<http://schema.org/';

function UID (len) {
    len = len || 23;
    let i = 0, random = '';
    for (; i < len; ++i) {
        random += '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'[0 | Math.random() * 62];
    }
    return '_:' + Hash('md5').update(random).digest('hex');
}

function inArray(array, id) {
    for (let i = 0, l = array.length; i < l; ++i) {
        if (array[i].id === id) {
            return true;
        }
    }
}

function nodesExists (client, nodes, callback) {
    client.g.V.apply(client.g, nodes).GetLimit(nodes.length, (err, result) => {

        if (err || (result && result.error)) {
            return callback(err || result.error);
        } 

        const exists = [];
        const inexistent = [];

        if (!result) {
            return callback(null, exists, nodes);
        }

        nodes.forEach(node => {
            if (inArray(result, node)) {
                exists.push(node);
            } else {
                inexistent.push(node);
            }
        });

        callback(null, exists, inexistent);
    });
}

exports.createNodeOfType = (client, type) => {

    const node = UID();

    switch (type) {
        case "onError":
        case "onEnd":
            type = 'sequence';
            break;
        case "next":
            // TODO handler type not in data
            // TODO add seq > handler > data/emit/stream
            type = 'data';
        break;
    }

    type = type.charAt(0).toUpperCase() + type.slice(1);

    return {
        uid: node,
        triples: [{
            subject: node,
            predicate: RDF_SYNTAX + 'type>',
            object: FLOW_VOCAB + type + '>'
        }]
    };
};

exports.ensureHashs = (client, values, callback) => {

    const hashs = [];
    values.forEach((value, index) => {
        hashs.push('_:' + Hash('md5').update(value).digest('hex'));
    });

    nodesExists(client, hashs, (err, exists, inexistent) => {

        if (err) {
            return callback(err);
        }

        if (inexistent.length === 0) {
            return callback(null, hashs);
        }

        const triples = [];
        inexistent.forEach((hash) => {
            triples.push({
                subject: hash,
                predicate: RDF_SYNTAX + 'string>',
                object: values[hashs.indexOf(hash)]
            });
        });

        callback(null, hashs, triples);
    });
};

exports.removeOut = (client, node, edges, callback) => {

    // TODO delete nodes with no in edges
    edges.forEach((edge, i) => {
        switch (edge) {
            case "name":
                edges[i] = SCHEMA + edge;
                break;
            case "type":
            case "string":
                edges[i] = RDF_SYNTAX + edge;
                break;
            default:
                edges[i] = FLOW_VOCAB + edge;
        }

        edges[i] += '>';
    });

    client.g.V(node).Tag('subject').Out(edges, 'predicate').All((err, result) => {

        if (err || !result || typeof result === 'string') {
            return callback(err || result);
        }

        const triples = [];
        result.forEach((triple) => {
            triples.push({
                subject: triple.subject,
                predicate: triple.predicate,
                object: triple.id
            });
        });

        client.delete(triples, (err, result) => {

            if (err || !result || typeof result === 'string') {
                return callback(err || result);
            }

            callback();
        });
    }); 
};
{'fraction_non_alphanumeric': 0.15785487047417057, 'fraction_numerical': 0.0093925162854113, 'mean_word_length': 1.7013093289689034, 'pattern_counts': {'":': 9, '<': 13, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7079445', 'n_tokens_mistral': 2136, 'n_tokens_neox': 2014, 'n_words': 433}