<commit_before>define(['rz_core', 'model/core', 'model/util', 'model/diff', 'consts', 'util'],
function(rz_core,   model_core,   model_util,   model_diff,   consts,   util) {
"use strict";

// Constants
var node_edge_separator = false;
var separator_symbol = '#'; //'  ';

var typeindex = 0,
    nodetypes = consts.nodetypes,
    node_name_to_type = {};

var _get_lastnode,
    get_lastnode = function (editgraph, cursor) {
        return undefined !== _get_lastnode ? _get_lastnode(editgraph, cursor) : null;
    };

var sugg_name = {},
    id_to_name_map = {},
    suggestions_bus = new Bacon.Bus(),
    suggestions_options = suggestions_bus.toProperty();

suggestions_bus.push([]);

var ANALYSIS_NODE_START = 'ANALYSIS_NODE_START';
var ANALYSIS_NODE = 'ANALYSIS_NODE';
var ANALYSIS_LINK = 'ANALYSIS_LINK';

var NEW_NODE_NAME = consts.NEW_NODE_NAME;

function selectedType()
{
    return nodetypes[typeindex];
}

function mod_2_second(mod)
{
    return function(_, i) { return i % 2 === mod; };
}

function even_second(_, i)
{
    return i % 2 === 0;
}

function list_first_pred(list, default_, pred)
{
    for (var i = 0 ; i < list.length ; ++i) {
        if (pred(list[i])) {
            return list[i];
        }
    }
    return default_;
}

function list_product(l1, l2)
{
    var ret = [],
        i, j;
    for (i = 0 ; i < l1.length ; ++i) {
        for (j = 0; j < l2.length ; ++j) {
            ret.push([l1[i], l2[j]]);
        }
    }
    return ret;
}

function subsets(N, jump, list)
{
    var ret = [],
        i;
    for (i = 0 ; i < list.length ; i += jump) {
        ret.push(list.slice(i, i + N));
    }
    return ret;
}

function subsets_strict(N, jump, list)
{
    return subsets(N, jump, list).filter(function (data) { return data.length === N; });
}

function group(list, predicate)
{
    if (list.length === 0) {
        return [];
    }
    var ret = [[list[0]]],
        i,
        last = predicate(list[0], 0),
        new_pred;

    for (i = 1 ; i < list.length ; ++i) {
        new_pred = predicate(list[i], i);
        if (new_pred === last) {
            ret[ret.length - 1].push(list[i]);
        } else {
            ret.push([list[i]]);
            last = new_pred;
        }
    }
    return ret;
}

function reduce(list, initial, func)
{
    var ret = initial;

    for (var i = 0 ; i < list.length; ++i) {
        ret = func(ret, list[i]);
    }
    return ret;
}

function list_get(list, index, def)
{
    var item = list[index];

    if (item === undefined) {
        return def;
    }
    return item;
}

function obj_take(name) {
    return function(obj) {
        return obj[name];
    };
}

/**
 *
 * tokens_to_graph_elements_*
 *
 * @param tokens: [token]
 * @ret [prefix, node, edge, node, edge, ..]
 * where
 *  node, edge = {start: int, end: int, tokens: [token]}
 */
function tokens_to_graph_elements_with_separator(tokens) {
    return reduce(tokens, [[] /* no prefix */, []], function (data, token) {
        if (token.token === separator_symbol) {
            data.push([]);
        } else {
            data[data.length - 1].push(token);
        }
        return data;
    });
}

function tokens_to_graph_elements_with_node_sign(tokens) {
    var ret = [[]]; // prefix is the first element
    for (var i = 0 ; i < tokens.length;) {
        if (tokens[i].token === separator_symbol) {
            if (tokens.length == i + 1) {
                tokens.push({end: tokens[i].end + 1 + NEW_NODE_NAME.length,
                             token: NEW_NODE_NAME});
            }
            tokens[i + 1].start = tokens[i].start; // used later for finding node from cursor position
            ret.push([tokens[i + 1]]);
            ret.push([]);
            i += 2;
        } else {
            ret[ret.length - 1].push(tokens[i]);
            i += 1;
        }
    }
    // remove last empty element if not just prefix - can only be a non empty link or a non empty node
    return ret.length > 1 && ret[ret.length - 1].length === 0 ? ret.slice(0, ret.length - 1) : ret;
}

var tokens_to_graph_elements;

if (node_edge_separator) {
    tokens_to_graph_elements = tokens_to_graph_elements_with_separator;
} else {
    tokens_to_graph_elements = tokens_to_graph_elements_with_node_sign;
}

function groups_to_thirds(groups) {
    return groups.map(function(tokens) {
        var start = tokens[0].start,
            end = tokens[tokens.length - 1].end;
        return {
            start: start,
            end: end,
            token: tokens.map(obj_take('token')).join(' ')
        };
    });<commit_msg>client/textanalysis: fix empty string case<commit_after>define(['rz_core', 'model/core', 'model/util', 'model/diff', 'consts', 'util'],
function(rz_core,   model_core,   model_util,   model_diff,   consts,   util) {
"use strict";

// Constants
var node_edge_separator = false;
var separator_symbol = '#'; //'  ';

var typeindex = 0,
    nodetypes = consts.nodetypes,
    node_name_to_type = {};

var _get_lastnode,
    get_lastnode = function (editgraph, cursor) {
        return undefined !== _get_lastnode ? _get_lastnode(editgraph, cursor) : null;
    };

var sugg_name = {},
    id_to_name_map = {},
    suggestions_bus = new Bacon.Bus(),
    suggestions_options = suggestions_bus.toProperty();

suggestions_bus.push([]);

var ANALYSIS_NODE_START = 'ANALYSIS_NODE_START';
var ANALYSIS_NODE = 'ANALYSIS_NODE';
var ANALYSIS_LINK = 'ANALYSIS_LINK';

var NEW_NODE_NAME = consts.NEW_NODE_NAME;

function selectedType()
{
    return nodetypes[typeindex];
}

function mod_2_second(mod)
{
    return function(_, i) { return i % 2 === mod; };
}

function even_second(_, i)
{
    return i % 2 === 0;
}

function list_first_pred(list, default_, pred)
{
    for (var i = 0 ; i < list.length ; ++i) {
        if (pred(list[i])) {
            return list[i];
        }
    }
    return default_;
}

function list_product(l1, l2)
{
    var ret = [],
        i, j;
    for (i = 0 ; i < l1.length ; ++i) {
        for (j = 0; j < l2.length ; ++j) {
            ret.push([l1[i], l2[j]]);
        }
    }
    return ret;
}

function subsets(N, jump, list)
{
    var ret = [],
        i;
    for (i = 0 ; i < list.length ; i += jump) {
        ret.push(list.slice(i, i + N));
    }
    return ret;
}

function subsets_strict(N, jump, list)
{
    return subsets(N, jump, list).filter(function (data) { return data.length === N; });
}

function group(list, predicate)
{
    if (list.length === 0) {
        return [];
    }
    var ret = [[list[0]]],
        i,
        last = predicate(list[0], 0),
        new_pred;

    for (i = 1 ; i < list.length ; ++i) {
        new_pred = predicate(list[i], i);
        if (new_pred === last) {
            ret[ret.length - 1].push(list[i]);
        } else {
            ret.push([list[i]]);
            last = new_pred;
        }
    }
    return ret;
}

function reduce(list, initial, func)
{
    var ret = initial;

    for (var i = 0 ; i < list.length; ++i) {
        ret = func(ret, list[i]);
    }
    return ret;
}

function list_get(list, index, def)
{
    var item = list[index];

    if (item === undefined) {
        return def;
    }
    return item;
}

function obj_take(name) {
    return function(obj) {
        return obj[name];
    };
}

function list_length_larger(n) {
    return function(list) {
        return list.length > n;
    }
}

/**
 *
 * tokens_to_graph_elements_*
 *
 * @param tokens: [token]
 * @ret [prefix, node, edge, node, edge, ..]
 * where
 *  node, edge = {start: int, end: int, tokens: [token]}
 */
function tokens_to_graph_elements_with_separator(tokens) {
    return reduce(tokens, [[] /* no prefix */, []], function (data, token) {
        if (token.token === separator_symbol) {
            data.push([]);
        } else {
            data[data.length - 1].push(token);
        }
        return data;
    });
}

function tokens_to_graph_elements_with_node_sign(tokens) {
    var ret = [[]]; // prefix is the first element
    for (var i = 0 ; i < tokens.length;) {
        if (tokens[i].token === separator_symbol) {
            if (tokens.length == i + 1) {
                tokens.push({end: tokens[i].end + 1 + NEW_NODE_NAME.length,
                             token: NEW_NODE_NAME});
            }
            tokens[i + 1].start = tokens[i].start; // used later for finding node from cursor position
            ret.push([tokens[i + 1]]);
            ret.push([]);
            i += 2;
        } else {
            ret[ret.length - 1].push(tokens[i]);
            i += 1;
        }
    }
    // remove last empty element if not just prefix - can only be a non empty link or a non empty node
    return ret.length > 1 && ret[ret.length - 1].length === 0 ? ret.slice(0, ret.length - 1) : ret;
}

var tokens_to_graph_elements;

if (node_edge_separator) {
    tokens_to_graph_elements = tokens_to_graph_elements_with_separator;
} else {
    tokens_to_graph_elements = tokens_to_graph_elements_with_node_sign;
}

function groups_to_thirds(groups) {
    return groups.filter(list_length_larger(0)).map(function(tokens) {
        var start = tokens[0].start,
            end = tokens[tokens.length - 1].end;
        return {
            start: start,
            end: end,
            token: tokens.map(obj_take('token')).join(' ')
        };
    });
{'fraction_non_alphanumeric': 0.1489476524554776, 'fraction_numerical': 0.008310847274689693, 'mean_word_length': 2.303386809269162, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6481431', 'n_tokens_mistral': 3241, 'n_tokens_neox': 3138, 'n_words': 743}