<commit_before>#!/usr/bin/env bash
# -*- coding: utf-8 -*-
#--------------------------
##	@Synopsis	Bash modular application framework.
##	@Copyright	Copyright 2009, <NAME>
##	@License	Apache
##
## The role of module.sh is to traverse application repository directories
## and try to load modules. Loading a module consist of:
## - finding source.sh,
## - sourcing source.sh,
## - running yourmodule_pre_source()
## - running yourmodule_source()
## - running yourmodule_post_source()
##
## Modules are also able to blacklist themselves.
##
## In order to find modules, this script expects the environment variable
## $MODULES_PATH to be set just like $PATH.
##
## Role of source.sh:
## Each module should have a source.sh file which may contain this functions:
## - yourmodule_pre_source(): basically set up variables needed by _source()
## - yourmodule_source(): include all dependencies
## - yourmodule_post_source(): initialise the module, ie. default variables,
##   conf path, whatever...
##
## Eventually, your module can have a yourmodule() function defined in source.sh
## to something meaningful. Think of it as an object constructor.
##
## Submodules are simply subdirectories of modules which contain source.sh. See
## module_pre_source() for more info.
#--------------------------

# Check bash version. We need at least 4.0.x {{{
# Lets not use anything like =~ here because
# that may not work on old bash versions.
if [[ "$(awk -F. '{print $1 $2}' <<< $BASH_VERSION)" -lt 40 ]]; then
	echo "Sorry your bash version is too old!"
	echo "You need at least version 3.2 of bash"
	echo "Please install a newer version:"
	echo " * Either use your distro's packages"
	echo " * Or see http://www.gnu.org/software/bash/"
	return 2
fi
# }}}
if [[ -z $MODULES_PATH ]]; then # {{{ return if $MODULES_PATH is not defined
    echo "Sorry, MODULES_PATH is not defined"
    echo "MODULES_PATH should contain a list of paths like the PATH variable"
    echo "It should contain a list of directories containing modules"
    echo "Each directory should be separated by ':'"
    echo "A module should be a folder with a file called 'source.sh' in it."
	return 2
fi
# }}}

# string module name => string module absolute patH
declare -A module_paths
# list of module names
declare -a module_blacklist

function module() {
    module_pre_source $*
    module_source
    module_post_source
}

#--------------------------
## This function finds all modules and nested submodules in a given path and
## registers it.
##
## With this example layout:
## /yourpath/
## /yourpath/foo/
## /yourpath/foo/source.sh
## /yourpath/foo/bar/source.sh
##
## It will register:
## module "foo" with path "/yourpath/foo"
## module "foo_bar" with path "/yourpath/foo/bar"
##
## It that example case, foo_bar functions should be prefixed by foo_bar_
## instead of just foo_.
## @Param   Path to find modules in.
#--------------------------
function module_pre_source() {
    if [[ -z $1 ]]; then
        # make an array of MODULES_PATH
        declare -a paths=($(echo $MODULES_PATH | tr : " "))
    else
        declare -a paths=()

        for path in $*; do
            paths+=("$path")
        done
    fi

    local module_name=""
    local relative_path=""

    # register modules and paths for each path
    for path in ${paths[@]}; do        
    
        for module_path in `find $path -name source.sh -exec dirname {} \;`; do
            relative_path="${module_path#*$path/}"
            module_name="${relative_path//\//_}"
            
            # blacklist check
            if [[ $(module_blacklist_check $module_name) ]]; then
                continue
            fi
            
            # add to module_path
            module_paths[$module_name]=$module_path
    
        done
    done
}

#--------------------------
## Resets module variables (ie. $module_paths and $module_blacklist).
## Traverses $MODULES_PATH searching for modules (directories with source.sh)
## Runs modulename_source() if it exists.
## Addes module names and paths to $module_paths.
## This function checks if the module was blacklisted at each step.
## @Param   Modules paths, default is $MODULES_PATH
## @Globals module_paths, module_blacklist
#--------------------------
function module_source() {
    local module_name=""
    local module_path=""<commit_msg>Make module_pre_source() to be reuseable during the same shell session by adding the module name->path key->value to module_paths variable only if it's not already there<commit_after>#!/usr/bin/env bash
# -*- coding: utf-8 -*-
#--------------------------
##	@Synopsis	Bash modular application framework.
##	@Copyright	Copyright 2009, <NAME>
##	@License	Apache
##
## The role of module.sh is to traverse application repository directories
## and try to load modules. Loading a module consist of:
## - finding source.sh,
## - sourcing source.sh,
## - running yourmodule_pre_source()
## - running yourmodule_source()
## - running yourmodule_post_source()
##
## Modules are also able to blacklist themselves.
##
## In order to find modules, this script expects the environment variable
## $MODULES_PATH to be set just like $PATH.
##
## Role of source.sh:
## Each module should have a source.sh file which may contain this functions:
## - yourmodule_pre_source(): basically set up variables needed by _source()
## - yourmodule_source(): include all dependencies
## - yourmodule_post_source(): initialise the module, ie. default variables,
##   conf path, whatever...
##
## Eventually, your module can have a yourmodule() function defined in source.sh
## to something meaningful. Think of it as an object constructor.
##
## Submodules are simply subdirectories of modules which contain source.sh. See
## module_pre_source() for more info.
#--------------------------

# Check bash version. We need at least 4.0.x {{{
# Lets not use anything like =~ here because
# that may not work on old bash versions.
if [[ "$(awk -F. '{print $1 $2}' <<< $BASH_VERSION)" -lt 40 ]]; then
	echo "Sorry your bash version is too old!"
	echo "You need at least version 3.2 of bash"
	echo "Please install a newer version:"
	echo " * Either use your distro's packages"
	echo " * Or see http://www.gnu.org/software/bash/"
	return 2
fi
# }}}
if [[ -z $MODULES_PATH ]]; then # {{{ return if $MODULES_PATH is not defined
    echo "Sorry, MODULES_PATH is not defined"
    echo "MODULES_PATH should contain a list of paths like the PATH variable"
    echo "It should contain a list of directories containing modules"
    echo "Each directory should be separated by ':'"
    echo "A module should be a folder with a file called 'source.sh' in it."
	return 2
fi
# }}}

# string module name => string module absolute patH
declare -A module_paths
# list of module names
declare -a module_blacklist

function module() {
    module_pre_source $*
    module_source
    module_post_source
}

#--------------------------
## This function finds all modules and nested submodules in a given path and
## registers it.
##
## With this example layout:
## /yourpath/
## /yourpath/foo/
## /yourpath/foo/source.sh
## /yourpath/foo/bar/source.sh
##
## It will register:
## module "foo" with path "/yourpath/foo"
## module "foo_bar" with path "/yourpath/foo/bar"
##
## It that example case, foo_bar functions should be prefixed by foo_bar_
## instead of just foo_.
## @Param   Path to find modules in.
#--------------------------
function module_pre_source() {
    if [[ -z $1 ]]; then
        # make an array of MODULES_PATH
        declare -a paths=($(echo $MODULES_PATH | tr : " "))
    else
        declare -a paths=()

        for path in $*; do
            paths+=("$path")
        done
    fi

    local module_name=""
    local relative_path=""

    # register modules and paths for each path
    for path in ${paths[@]}; do        
    
        for module_path in `find $path -name source.sh -exec dirname {} \;`; do
            relative_path="${module_path#*$path/}"
            module_name="${relative_path//\//_}"
            
            # blacklist check
            if [[ $(module_blacklist_check $module_name) ]]; then
                continue
            fi
            
            # add to module_path if required
            if [[ ! "${module_paths[@]}" =~ "$module_path" ]]; then
                module_paths[$module_name]=$module_path
            fi
        
        done
    done
}

#--------------------------
## Resets module variables (ie. $module_paths and $module_blacklist).
## Traverses $MODULES_PATH searching for modules (directories with source.sh)
## Runs modulename_source() if it exists.
## Addes module names and paths to $module_paths.
## This function checks if the module was blacklisted at each step.
## @Param   Modules paths, default is $MODULES_PATH
## @Globals module_paths, module_blacklist
#--------------------------
function module_source() {
    local module_name=""
    local module_path=""
{'fraction_non_alphanumeric': 0.14239264491534925, 'fraction_numerical': 0.0035878461710954143, 'mean_word_length': 2.7828668363019506, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1481068', 'n_tokens_mistral': 2550, 'n_tokens_neox': 2494, 'n_words': 978}