<commit_before>/*
This file is part of Bohrium and copyright (c) 2012 the Bohrium
team <http://www.bh107.org>.

Bohrium is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as 
published by the Free Software Foundation, either version 3 
of the License, or (at your option) any later version.

Bohrium is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the 
GNU Lesser General Public License along with Bohrium. 

If not, see <http://www.gnu.org/licenses/>.
*/

#include <cassert>
#include <cstring>
#include <iostream>
#include <vector>
#include <set>
#include <bh.h>

#include "bh_vem_cluster.h"
#include "mapping.h"
#include "array.h"
#include "pgrid.h"
#include "dispatch.h"
#include "comm.h"
#include "except.h"
#include "ufunc_reduce.h"
#include "batch.h"
#include "tmp.h"


//Function pointers to the Node VEM.
static bh_init vem_init;
static bh_shutdown vem_shutdown;
static bh_reg_func vem_reg_func;
//Public function pointer to the Node VEM
bh_execute exec_vem_execute;

//The VE components
static bh_component **my_components;

//Our self
static bh_component *myself;

//Number of user-defined functions registered.
static bh_intp userfunc_count = 0;
//User-defined function IDs.
static bh_userfunc_impl reduce_impl = NULL;
static bh_intp reduce_impl_id = 0;


/* Initialize the VEM
 *
 * @return Error codes (BH_SUCCESS)
 */
bh_error exec_init(const char *component_name)
{
    bh_intp children_count;
    bh_error err;
    myself = bh_component_setup(component_name);
    if(myself == NULL)
        return BH_ERROR;
    
    err = bh_component_children(myself, &children_count, &my_components);
    if (children_count != 1) 
    {
		std::cerr << "Unexpected number of child nodes for VEM, must be 1" << std::endl;
		return BH_ERROR;
    }
    
    if (err != BH_SUCCESS)
	    return err;
    
    vem_init = my_components[0]->init;
    exec_vem_execute = my_components[0]->execute;
    vem_shutdown = my_components[0]->shutdown;
    vem_reg_func = my_components[0]->reg_func;

    //Let us initiate the Node VEM.
    if((err = vem_init(my_components[0])) != 0)
        return err;

    return BH_SUCCESS;
}


/* Shutdown the VEM, which include a instruction flush
 *
 * @return Error codes (BH_SUCCESS)
 */
bh_error exec_shutdown(void)
{
    bh_error err;
    if((err = vem_shutdown()) != BH_SUCCESS)
        return err;
    bh_component_free(my_components[0]);//Only got one child.
    vem_init     = NULL;
    exec_vem_execute  = NULL;
    vem_shutdown = NULL;
    vem_reg_func = NULL;
    bh_component_free_ptr(my_components);
    my_components = NULL;

    //Finalize the process grid
    pgrid_finalize();

    //Finalize the process grid
    dispatch_finalize();

    return BH_SUCCESS;
}


/* Register a new user-defined function.
 *
 * @lib Name of the shared library e.g. libmyfunc.so
 *      When NULL the default library is used.
 * @fun Name of the function e.g. myfunc
 * @id Identifier for the new function. The bridge should set the
 *     initial value to Zero. (in/out-put)
 * @return Error codes (BH_SUCCESS)
 */
bh_error exec_reg_func(char *fun, bh_intp *id)
{
    bh_error e;
    
    if(*id == 0)//Only if parent didn't set the ID.
    {
        *id = ++userfunc_count;
        assert(pgrid_myrank == 0);
    }

    if((e = vem_reg_func(fun, id)) != BH_SUCCESS)
    {
        *id = 0;
        return e;
    }

    if(strcmp("bh_reduce", fun) == 0)
    {
        if(reduce_impl == NULL)
        {
            bh_component_get_func(myself, fun, &reduce_impl);
            if (reduce_impl == NULL)
                return BH_USERFUNC_NOT_SUPPORTED;

            reduce_impl_id = *id;
            return BH_SUCCESS;           
        }
    }

    return BH_SUCCESS;
}


/* Execute to instruction locally at the master-process
 *
 * @instruction The instructionto execute
 */
static void fallback_exec(bh_instruction *inst)
{
    int nop = bh_operands_in_instruction(inst);
    std::set<bh_array*> arys2discard;
    
    batch_flush();

    //Gather all data at the master-process
    bh_array **oprands = bh_inst_operands(inst);
    for(bh_intp o=0; o < nop; ++o)
    {<commit_msg>Implemented a fallback counter for the CLUSTER-VEM<commit_after>/*
This file is part of Bohrium and copyright (c) 2012 the Bohrium
team <http://www.bh107.org>.

Bohrium is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as 
published by the Free Software Foundation, either version 3 
of the License, or (at your option) any later version.

Bohrium is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the 
GNU Lesser General Public License along with Bohrium. 

If not, see <http://www.gnu.org/licenses/>.
*/

#include <cassert>
#include <cstring>
#include <iostream>
#include <vector>
#include <set>
#include <bh.h>

#include "bh_vem_cluster.h"
#include "mapping.h"
#include "array.h"
#include "pgrid.h"
#include "dispatch.h"
#include "comm.h"
#include "except.h"
#include "ufunc_reduce.h"
#include "batch.h"
#include "tmp.h"


//Function pointers to the Node VEM.
static bh_init vem_init;
static bh_shutdown vem_shutdown;
static bh_reg_func vem_reg_func;
//Public function pointer to the Node VEM
bh_execute exec_vem_execute;

//The VE components
static bh_component **my_components;

//Our self
static bh_component *myself;

//Number of user-defined functions registered.
static bh_intp userfunc_count = 0;
//User-defined function IDs.
static bh_userfunc_impl reduce_impl = NULL;
static bh_intp reduce_impl_id = 0;

//Number of instruction fallbacks
static bh_intp fallback_count = 0;


/* Initialize the VEM
 *
 * @return Error codes (BH_SUCCESS)
 */
bh_error exec_init(const char *component_name)
{
    bh_intp children_count;
    bh_error err;
    myself = bh_component_setup(component_name);
    if(myself == NULL)
        return BH_ERROR;
    
    err = bh_component_children(myself, &children_count, &my_components);
    if (children_count != 1) 
    {
		std::cerr << "Unexpected number of child nodes for VEM, must be 1" << std::endl;
		return BH_ERROR;
    }
    
    if (err != BH_SUCCESS)
	    return err;
    
    vem_init = my_components[0]->init;
    exec_vem_execute = my_components[0]->execute;
    vem_shutdown = my_components[0]->shutdown;
    vem_reg_func = my_components[0]->reg_func;

    //Let us initiate the Node VEM.
    if((err = vem_init(my_components[0])) != 0)
        return err;

    return BH_SUCCESS;
}


/* Shutdown the VEM, which include a instruction flush
 *
 * @return Error codes (BH_SUCCESS)
 */
bh_error exec_shutdown(void)
{
    bh_error err;
    if((err = vem_shutdown()) != BH_SUCCESS)
        return err;
    bh_component_free(my_components[0]);//Only got one child.
    vem_init     = NULL;
    exec_vem_execute  = NULL;
    vem_shutdown = NULL;
    vem_reg_func = NULL;
    bh_component_free_ptr(my_components);
    my_components = NULL;

    //Finalize the process grid
    pgrid_finalize();

    //Finalize the process grid
    dispatch_finalize();

    if(fallback_count > 0)
        fprintf(stderr, "[CLUSTER-VEM] Warning - fallen back to "
        "sequential executing %ld times.\n", fallback_count);

    return BH_SUCCESS;
}


/* Register a new user-defined function.
 *
 * @lib Name of the shared library e.g. libmyfunc.so
 *      When NULL the default library is used.
 * @fun Name of the function e.g. myfunc
 * @id Identifier for the new function. The bridge should set the
 *     initial value to Zero. (in/out-put)
 * @return Error codes (BH_SUCCESS)
 */
bh_error exec_reg_func(char *fun, bh_intp *id)
{
    bh_error e;
    
    if(*id == 0)//Only if parent didn't set the ID.
    {
        *id = ++userfunc_count;
        assert(pgrid_myrank == 0);
    }

    if((e = vem_reg_func(fun, id)) != BH_SUCCESS)
    {
        *id = 0;
        return e;
    }

    if(strcmp("bh_reduce", fun) == 0)
    {
        if(reduce_impl == NULL)
        {
            bh_component_get_func(myself, fun, &reduce_impl);
            if (reduce_impl == NULL)
                return BH_USERFUNC_NOT_SUPPORTED;

            reduce_impl_id = *id;
            return BH_SUCCESS;           
        }
    }

    return BH_SUCCESS;
}


/* Execute to instruction locally at the master-process
 *
 * @instruction The instructionto execute
 */
static void fallback_exec(bh_instruction *inst)
{
    int nop = bh_operands_in_instruction(inst);
    std::set<bh_array*> arys2discard;
    
    batch_flush();

    ++fallback_count;

    //Gather all data at the master-process
    bh_array **oprands = bh_inst_operands(inst);
    for(bh_intp o=0; o < nop; ++o)
    {
{'fraction_non_alphanumeric': 0.0996020340481981, 'fraction_numerical': 0.00574839708158302, 'mean_word_length': 3.1595402298850574, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1031119', 'n_tokens_mistral': 3118, 'n_tokens_neox': 2978, 'n_words': 916}