<commit_before>define([
    'coreclient/meta',
    'util/url'
],function(
    META,
    URL
    ){

    var _refTypes = {
        'url':'url',
        'path':'path',
        'guid':'guid'
    };

    var changeRefObjects = function(refType,urlPrefix,object,core,root,callback){
        if(typeof object === 'object'){
            var needed = 0,
                neededNames = [],
                error = null;
            for(var i in object){
                if(typeof object[i] === 'object'){
                    needed++;
                    neededNames.push(i);
                }
            }
            if(needed > 0){
                for(var i=0;i<neededNames.length;i++){
                    if(object[neededNames[i]]['$ref']){
                        //refrence object
                        pathToRefObjAsync(refType,urlPrefix,object[neededNames[i]]['$ref'].substring(1),core,root,function(err,refObj){
                            error = error || err;
                            object[neededNames[i]] = refObj;
                            if(--needed === 0){
                                callback(error);
                            }
                        });
                    } else {
                        changeRefObjects(refType,urlPrefix,object[neededNames[i]],core,root,function(err){
                            error = error || err;
                            if(--needed === 0){
                                callback(error);
                            }
                        });
                    }
                }
            } else {
                callback(null);
            }
        } else {
            callback(null);
        }
    };
    var pathToRefObj = function(refType,urlPrefix,path){
        switch (refType){
            case _refTypes.url:
                if(path === null){
                    return URL.urlToRefObject(null);
                }
                return URL.urlToRefObject(urlPrefix+'/'+URL.addSpecialChars(path));
                break;
            case _refTypes.path:
                return URL.urlToRefObject(path);
                break;
            default:
                return URL.urlToRefObject(null);
        }
    };
    var pathToRefObjAsync = function(refType,urlPrefix,path,core,root,callback){
        switch (refType){
            case _refTypes.url:
                if(path === null){
                    callback(null,URL.urlToRefObject(null));
                }
                callback(null,URL.urlToRefObject(urlPrefix+'/'+URL.addSpecialChars(path)));
                break;
            case _refTypes.path:
                callback(null,URL.urlToRefObject(path));
                break;
            case _refTypes.guid:
                core.loadByPath(root,path,function(err,node){
                    if(err){
                        callback(err,null);
                    } else {
                        var refObj = URL.urlToRefObject(path);
                        refObj.GUID = core.getGuid(node);
                        callback(null,refObj);
                    }
                });
                break;
            default:
                callback(null,URL.urlToRefObject(null));
        }
    };
    var getChildrenGuids = function(core,node,callback){
        var GUIDHash = {};
        core.loadChildren(node,function(err,children){
            if(err){
                callback(err,null);
            } else {
                for(var i=0;i<children.length;i++){
                    GUIDHash[core.getPath(children[i])] = core.getGuid(children[i]);
                }
                callback(null,GUIDHash);
            }
        });
    };
    var getMetaOfNode = function(core,node,urlPrefix,refType,callback){
        var meta = META.getMeta(core.getPath(node));
        changeRefObjects(refType,urlPrefix,meta,core,core.getRoot(node),function(err){
            callback(err,meta);
        });
    };
    var getChildrenOfNode = function(core,node,urlPrefix,refType,callback){
        if(refType === _refTypes.guid){
            getChildrenGuids(core,node,function(err,gHash){
                if(err){
                    callback(err);
                } else {
                    //TODO possibly it needs some ordering
                    var children = [];
                    for(var i in gHash){
                        var refObj = URL.urlToRefObject(i);
                        refObj.GUID = gHash[i];
                        children.push(refObj);
                    }
                    callback(null,children);
                }
            });
        } else {
            var paths = core.getChildrenPaths(node);
            var children = [];
            for(var i=0;i<paths.length;i++){
                children.push(pathToRefObj(refType,urlPrefix,paths[i]));
            }
            callback(null,children);
        }
    };
    var getSetsOfNode = function(core,node,urlPrefix,refType,callback){
        var setsInfo = {};
        var createOneSetInfo = function(setName,callback){
            var needed,
                members = core.getMemberPaths(node,setName),
                info = {from:[],to:[]},
                i,
                error = null,
                containers = [];

            for(i in memberOfInfo){
                if(memberOfInfo[i].indexOf(setName) !== -1){
                    containers.push(core.toActualPath(i));
                }
            }

            needed = members.length + containers.length;
            if(needed > 0){
                for(i=0;i<members.length;i++){
                    pathToRefObjAsync(refType,urlPrefix,members[i],core,core.getRoot(node),function(err,refObj){
                        error = error || err;
                        if(refObj !== undefined && refObj !== null){
                            info.to.push(refObj);
                        }

                        if(--needed === 0){
                            if(error === null){
                                setsInfo[setName] = info;
                            }
                            callback(error);
                        }
                    });
                }

                for(i=0;i<containers.length;i++){
                    pathToRefObjAsync(refType,urlPrefix,containers[i],core,core.getRoot(node),function(err,refObj){
                        error = error || err;
                        if(refObj !== undefined && refObj !== null){
                            info.from.push(refObj);
                        }

                        if(--needed === 0){
                            if(error === null){
                                setsInfo[setName] = info;
                            }
                            callback(error);
                        }
                    });
                }
            } else {
                callback(null);
            }
        };

        var tArray = core.getSetNames(node),
            memberOfInfo = core.isMemberOf(node),
            i, j, needed, error = null;
        for(j in memberOfInfo){
            for(i=0;i<memberOfInfo[j].length;i++){
                if(tArray.indexOf(memberOfInfo[j][i]) === -1){
                    tArray.push(memberOfInfo[j][i]);
                }
            }
        }
        needed = tArray.length;
        if(needed>0){
            for(i=0;i<tArray.length;i++){
                createOneSetInfo(tArray[i],function(err){
                    error = error || err;
                    if(--needed === 0){
                        callback(error,setsInfo);<commit_msg>now the set attributes and registry have been added to the json format<commit_after>define([
    'coreclient/meta',
    'util/url'
],function(
    META,
    URL
    ){

    var _refTypes = {
        'url':'url',
        'path':'path',
        'guid':'guid'
    };

    var changeRefObjects = function(refType,urlPrefix,object,core,root,callback){
        if(typeof object === 'object'){
            var needed = 0,
                neededNames = [],
                error = null;
            for(var i in object){
                if(typeof object[i] === 'object'){
                    needed++;
                    neededNames.push(i);
                }
            }
            if(needed > 0){
                for(var i=0;i<neededNames.length;i++){
                    if(object[neededNames[i]]['$ref']){
                        //refrence object
                        pathToRefObjAsync(refType,urlPrefix,object[neededNames[i]]['$ref'].substring(1),core,root,function(err,refObj){
                            error = error || err;
                            object[neededNames[i]] = refObj;
                            if(--needed === 0){
                                callback(error);
                            }
                        });
                    } else {
                        changeRefObjects(refType,urlPrefix,object[neededNames[i]],core,root,function(err){
                            error = error || err;
                            if(--needed === 0){
                                callback(error);
                            }
                        });
                    }
                }
            } else {
                callback(null);
            }
        } else {
            callback(null);
        }
    };
    var pathToRefObj = function(refType,urlPrefix,path){
        switch (refType){
            case _refTypes.url:
                if(path === null){
                    return URL.urlToRefObject(null);
                }
                return URL.urlToRefObject(urlPrefix+'/'+URL.addSpecialChars(path));
                break;
            case _refTypes.path:
                return URL.urlToRefObject(path);
                break;
            default:
                return URL.urlToRefObject(null);
        }
    };
    var pathToRefObjAsync = function(refType,urlPrefix,path,core,root,callback){
        switch (refType){
            case _refTypes.url:
                if(path === null){
                    callback(null,URL.urlToRefObject(null));
                }
                callback(null,URL.urlToRefObject(urlPrefix+'/'+URL.addSpecialChars(path)));
                break;
            case _refTypes.path:
                callback(null,URL.urlToRefObject(path));
                break;
            case _refTypes.guid:
                core.loadByPath(root,path,function(err,node){
                    if(err){
                        callback(err,null);
                    } else {
                        var refObj = URL.urlToRefObject(path);
                        refObj.GUID = core.getGuid(node);
                        callback(null,refObj);
                    }
                });
                break;
            default:
                callback(null,URL.urlToRefObject(null));
        }
    };
    var getChildrenGuids = function(core,node,callback){
        var GUIDHash = {};
        core.loadChildren(node,function(err,children){
            if(err){
                callback(err,null);
            } else {
                for(var i=0;i<children.length;i++){
                    GUIDHash[core.getPath(children[i])] = core.getGuid(children[i]);
                }
                callback(null,GUIDHash);
            }
        });
    };
    var getMetaOfNode = function(core,node,urlPrefix,refType,callback){
        var meta = META.getMeta(core.getPath(node));
        changeRefObjects(refType,urlPrefix,meta,core,core.getRoot(node),function(err){
            callback(err,meta);
        });
    };
    var getChildrenOfNode = function(core,node,urlPrefix,refType,callback){
        if(refType === _refTypes.guid){
            getChildrenGuids(core,node,function(err,gHash){
                if(err){
                    callback(err);
                } else {
                    //TODO possibly it needs some ordering
                    var children = [];
                    for(var i in gHash){
                        var refObj = URL.urlToRefObject(i);
                        refObj.GUID = gHash[i];
                        children.push(refObj);
                    }
                    callback(null,children);
                }
            });
        } else {
            var paths = core.getChildrenPaths(node);
            var children = [];
            for(var i=0;i<paths.length;i++){
                children.push(pathToRefObj(refType,urlPrefix,paths[i]));
            }
            callback(null,children);
        }
    };
    var getSetAttributesAndRegistry = function(core,node,setName,setOwnerPath,callback){
        var path = core.getPath(node);
        core.loadByPath(core.getRoot(node),setOwnerPath,function(err,owner){
            if(err){
                callback(err);
            } else {
                if(owner){
                    var atrAndReg = {attributes:{},registry:{}};
                    var names = core.getMemberAttributeNames(owner,setName,path);
                    for(var i=0;i<names.length;i++){
                        atrAndReg.attributes[names[i]] = core.getMemberAttribute(owner,setName,path,names[i]);
                    }
                    names = core.getMemberRegistryNames(owner,setName,path);
                    for(var i=0;i<names.length;i++){
                        atrAndReg.registry[names[i]] = core.getMemberRegistry(owner,setName,path,names[i]);
                    }
                    callback(null,atrAndReg);
                } else {
                    callback('internal error',null);
                }
            }
        });
    };
    var getSetsOfNode = function(core,node,urlPrefix,refType,callback){
        var setsInfo = {};
        var createOneSetInfo = function(setName,callback){
            var needed,
                members = core.getMemberPaths(node,setName),
                info = {from:[],to:[],attributes:{},registry:{}},
                i,
                error = null,
                containers = [];

            for(i in memberOfInfo){
                if(memberOfInfo[i].indexOf(setName) !== -1){
                    containers.push(core.toActualPath(i));
                }
            }

            needed = members.length + containers.length;
            if(needed > 0){
                for(i=0;i<members.length;i++){
                    pathToRefObjAsync(refType,urlPrefix,members[i],core,core.getRoot(node),function(err,refObj){
                        error = error || err;
                        if(refObj !== undefined && refObj !== null){
                            info.to.push(refObj);
                        }

                        if(--needed === 0){
                            if(error === null){
                                setsInfo[setName] = info;
                            }
                            callback(error);
                        }
                    });
                }

                for(i=0;i<containers.length;i++){
                    getSetAttributesAndRegistry(core,node,setName,containers[i],function(err,atrAndReg){
                        error = error || err;
                        if(atrAndReg){
                            info.attributes[containers[i]] = atrAndReg.attributes;
                            info.registry[containers[i]] = atrAndReg.registry;
                        }
                        
                        pathToRefObjAsync(refType,urlPrefix,containers[i],core,core.getRoot(node),function(err,refObj){
                            error = error || err;
                            if(refObj !== undefined && refObj !== null){
                                info.from.push(refObj);
                            }

                            if(--needed === 0){
                                if(error === null){
                                    setsInfo[setName] = info;
                                }
                                callback(error);
                            }
                        });
                    });
                }
            } else {
                callback(null);
            }
        };

        var tArray = core.getSetNames(node),
            memberOfInfo = core.isMemberOf(node),
            i, j, needed, error = null;
        for(j in memberOfInfo){
            for(i=0;i<memberOfInfo[j].length;i++){
                if(tArray.indexOf(memberOfInfo[j][i]) === -1){
                    tArray.push(memberOfInfo[j][i]);
                }
            }
        }
        needed = tArray.length;
        if(needed>0){
            for(i=0;i<tArray.length;i++){
                createOneSetInfo(tArray[i],function(err){
                    error = error || err;
                    if(--needed === 0){
                        callback(error,setsInfo);
{'fraction_non_alphanumeric': 0.11659005432511492, 'fraction_numerical': 0.0023879171392752674, 'mean_word_length': 1.131840162891321, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6773446', 'n_tokens_mistral': 4386, 'n_tokens_neox': 4112, 'n_words': 583}