<commit_before>#
#    ICRAR - International Centre for Radio Astronomy Research
#    (c) UWA - The University of Western Australia, 2015
#    Copyright by UWA (in the framework of the ICRAR)
#    All rights reserved
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#    MA 02111-1307  USA
#

"""
Dropmake utils
"""

import json
import os
import os.path as osp

LG_VER_OLD = 1
LG_VER_EAGLE_CONVERTED = 2
LG_VER_EAGLE = 3

TEMP_FILE_FOLDER = '/tmp'

def get_lg_ver_type(lgo):
    """
    Get the version type of this logical graph
    """
    # with open(lg_name, 'r+') as file_object:
    #     lgo = json.load(file_object)

    nodes = lgo['nodeDataArray']
    if (len(nodes) == 0):
        raise Exception("Invalid LG, nodes not found")

    if ('fields' in nodes[0]):
        fds = nodes[0]['fields']
        kw = fds[0]['name']
        if (kw in nodes[0]):
            return LG_VER_EAGLE_CONVERTED
        else:
            return LG_VER_EAGLE

    if ('linkDataArray' in lgo and len(lgo['linkDataArray']) > 0):
        lnk = lgo['linkDataArray'][0]
        if ('fromPort' not in lnk):
            return LG_VER_OLD
        if (lnk['fromPort'] in ['B', 'T', 'R', 'L']):
            # only the old lg links can be Bottom, Top, Right and Left
            return LG_VER_OLD
        else:
            return LG_VER_EAGLE_CONVERTED
    else:
        if ('copiesArrayObjects' in lgo or 'copiesArrays' in lgo):
            return LG_VER_EAGLE_CONVERTED
        else:
            return LG_VER_OLD

def get_keyset(lgo):
    return set([x['key'] for x in lgo['nodeDataArray']])

def convert_fields(lgo):
    nodes = lgo['nodeDataArray']
    for node in nodes:
        fields = node['fields']
        for field in fields:
            name = field.get('name', '')
            if (name != ''):
                # Add a node property.
                # print("Set %s to %s" % (name, field.get('value', '')))
                node[name] = field.get('value', '')
    return lgo

def convert_construct(lgo):
    """
    1. for each scatter/gather, create a "new" application drop, which shares
       the same 'key' as the construct
    2. reset the key of the scatter/gather construct to 'k_new'
    3. reset the "group" keyword of each drop inside the construct to 'k_new'
    """
    #print('%d nodes in lg' % len(lgo['nodeDataArray']))
    keyset = get_keyset(lgo)
    old_new_grpk_map = dict()
    old_new_gather_map = dict()
    new_nodes = []
    for node in lgo['nodeDataArray']:
        if (node['category'] not in ['SplitData', 'DataGather']):
            continue
        if ('application' not in node):
            continue
        # step 1
        app_node = dict()
        app_node['key'] = node['key']
        app_node['category'] = node['application']
        if ('group' in node):
            app_node['group'] = node['group']
        if ('appFields' in node):
            for afd in node['appFields']:
                app_node[afd['name']] = afd['value']
        if ('DataGather' == node['category']):
            app_node['group_start'] = 1
        new_nodes.append(app_node)

        # step 2
        k_new = min(keyset) - 1
        node['key'] = k_new
        keyset.add(k_new)
        old_new_grpk_map[app_node['key']] = k_new

        if ('DataGather' == node['category']):
            old_new_gather_map[app_node['key']] = k_new

    if (len(new_nodes) > 0):
        lgo['nodeDataArray'].extend(new_nodes)<commit_msg>recover the first component inside the gather construct<commit_after>#
#    ICRAR - International Centre for Radio Astronomy Research
#    (c) UWA - The University of Western Australia, 2015
#    Copyright by UWA (in the framework of the ICRAR)
#    All rights reserved
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#    MA 02111-1307  USA
#

"""
Dropmake utils
"""

import json
import os
import os.path as osp

LG_VER_OLD = 1
LG_VER_EAGLE_CONVERTED = 2
LG_VER_EAGLE = 3

TEMP_FILE_FOLDER = '/tmp'

def get_lg_ver_type(lgo):
    """
    Get the version type of this logical graph
    """
    # with open(lg_name, 'r+') as file_object:
    #     lgo = json.load(file_object)

    nodes = lgo['nodeDataArray']
    if (len(nodes) == 0):
        raise Exception("Invalid LG, nodes not found")

    if ('fields' in nodes[0]):
        fds = nodes[0]['fields']
        kw = fds[0]['name']
        if (kw in nodes[0]):
            return LG_VER_EAGLE_CONVERTED
        else:
            return LG_VER_EAGLE

    if ('linkDataArray' in lgo and len(lgo['linkDataArray']) > 0):
        lnk = lgo['linkDataArray'][0]
        if ('fromPort' not in lnk):
            return LG_VER_OLD
        if (lnk['fromPort'] in ['B', 'T', 'R', 'L']):
            # only the old lg links can be Bottom, Top, Right and Left
            return LG_VER_OLD
        else:
            return LG_VER_EAGLE_CONVERTED
    else:
        if ('copiesArrayObjects' in lgo or 'copiesArrays' in lgo):
            return LG_VER_EAGLE_CONVERTED
        else:
            return LG_VER_OLD

def get_keyset(lgo):
    return set([x['key'] for x in lgo['nodeDataArray']])

def convert_fields(lgo):
    nodes = lgo['nodeDataArray']
    for node in nodes:
        fields = node['fields']
        for field in fields:
            name = field.get('name', '')
            if (name != ''):
                # Add a node property.
                # print("Set %s to %s" % (name, field.get('value', '')))
                node[name] = field.get('value', '')
    return lgo

def _build_node_index(lgo):
    ret = dict()
    for node in lgo['nodeDataArray']:
        ret[node['key']] = node

    return ret

def _relink_gather(appnode, lgo, gather_newkey, node_index):
    """
    for links whose 'from' is gather, 'to' is gather-internal data,
    relink them such that 'from' is the appnode
    """

    gather_oldkey = appnode['key']
    for link in lgo['linkDataArray']:
        if (link['from'] == gather_oldkey):
            node = node_index[link['to']]
            if (node['group'] == gather_oldkey):
                pass

def convert_construct(lgo):
    """
    1. for each scatter/gather, create a "new" application drop, which shares
       the same 'key' as the construct
    2. reset the key of the scatter/gather construct to 'k_new'
    3. reset the "group" keyword of each drop inside the construct to 'k_new'
    """
    #print('%d nodes in lg' % len(lgo['nodeDataArray']))
    keyset = get_keyset(lgo)
    old_new_grpk_map = dict()
    old_new_gather_map = dict()
    new_nodes = []
    #node_index = _build_node_index(lgo) # old index
    for node in lgo['nodeDataArray']:
        if (node['category'] not in ['SplitData', 'DataGather']):
            continue
        if ('application' not in node):
            continue
        # step 1
        app_node = dict()
        app_node['key'] = node['key']
        app_node['category'] = node['application']
        app_node['text'] = node['text']
        if ('group' in node):
            app_node['group'] = node['group']
        if ('appFields' in node):
            for afd in node['appFields']:
                app_node[afd['name']] = afd['value']
        if ('DataGather' == node['category']):
            app_node['group_start'] = 1
        new_nodes.append(app_node)

        # step 2
        k_new = min(keyset) - 1
        node['key'] = k_new
        keyset.add(k_new)
        old_new_grpk_map[app_node['key']] = k_new

        if ('DataGather' == node['category']):
            old_new_gather_map[app_node['key']] = k_new
            app_node['group'] = k_new
            app_node['group_start'] = 1

    if (len(new_nodes) > 0):
        lgo['nodeDataArray'].extend(new_nodes)
{'fraction_non_alphanumeric': 0.11509832240862126, 'fraction_numerical': 0.008554605043884013, 'mean_word_length': 2.454336147352264, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '481266', 'n_tokens_mistral': 2906, 'n_tokens_neox': 2818, 'n_words': 908}