<commit_before>#include <stdio.h>
#include <iostream>				// std::cerr
#include <iomanip>              // std::setw

#include <errno.h>

#include "logMsg/logMsg.h"				// LM_X

#include "au/string.h"             // au::Format
#include "au/CommandLine.h"		// au::CommandLine
#include "au/ErrorManager.h"           // au::ErrorManager

#include "samson/common/samsonDirectories.h"	// SAMSON_SETUP_FILE
#include "samson/common/SamsonSetup.h"		// Own interface

namespace samson
{
    
	void createDirectory( std::string path )
	{
		if( mkdir(path.c_str()	, 0755) == -1 )
		{
			if( errno != EEXIST )
				LM_X(1,("Error creating directory %s", path.c_str()));
		}
	}
    
    
#pragma mark SetupItemCollection
    
    void  SetupItemCollection::add( std::string _name , std::string _default_value , std::string _description )
    {
        if( items.findInMap(_name) != NULL )
        {
            LM_W(("Item %s already added to setup... ignoring" , _name.c_str() ));
            return;
        }
        
        items.insertInMap( _name , new SetupItem( _name , _default_value , _description) );
        
    }
    
    
    void SetupItemCollection::load( std::string fileName )
    {
		FILE *file = fopen( fileName.c_str()  ,"r");
		if (!file)
		{
            LM_W(("Warning: Setup file %s not found\n" , fileName.c_str() ));
		}
		else
		{
			
			char line[2000];
			while( fgets(line, sizeof(line), file))
			{
				au::CommandLine c;
				c.parse(line);
				
				if( c.get_num_arguments() == 0 )
					continue;
				
				// Skip comments
				std::string mainCommand = c.get_argument(0);
				if( mainCommand[0] == '#' )
					continue;
				
				if (c.get_num_arguments() >= 2)	
				{
					std::string name = c.get_argument(0);
					std::string value =  c.get_argument(1);
                    
                    SetupItem*item = items.findInMap(name);
                    
                    if( item )
                        item->setValue(value);
                    else
                        LM_W(("Parameter %s found in setup file %s not included since it is not defined." , name.c_str() , fileName.c_str() ));
                    
				}
			}
            
			fclose(file);
		}        
    }

    std::string SetupItemCollection::getValueForParameter( std::string name )
    {
        SetupItem*item = items.findInMap(name);
        
        if( !item )
            LM_X(1, ("Parameter %s not defined in the setup. This is not acceptable", name.c_str()));
        
        return item->getValue();
        
    }
    
    bool SetupItemCollection::isParameterDefined( std::string name )
    {
        return ( items.findInMap(name) != NULL);
    }
    
    
    void SetupItemCollection::setValueForParameter( std::string name ,std::string value )
    {
        SetupItem*item = items.findInMap(name);
        
        if( !item )
        {
            LM_W(("Parameter %s not defined in the setup. This is not acceptable", name.c_str()));
            return;
        }
        
        item->setValue( value );
    }
    
    std::string SetupItemCollection::str()
    {
        std::ostringstream output;
        output << "Samson setup parameters:\n";
        output << "---------------------------------------------------------------------------------------------------\n";

        output << std::left;
        
        // Heder
        output << "\n";
        output << std::setw(40) << "Parameter" << " ";
        output << std::setw(20) << "Default value" << " ";
        output << std::setw(20) << "Value at setup.txt" << " ";<commit_msg>in not at<commit_after>#include <stdio.h>
#include <iostream>				// std::cerr
#include <iomanip>              // std::setw

#include <errno.h>

#include "logMsg/logMsg.h"				// LM_X

#include "au/string.h"             // au::Format
#include "au/CommandLine.h"		// au::CommandLine
#include "au/ErrorManager.h"           // au::ErrorManager

#include "samson/common/samsonDirectories.h"	// SAMSON_SETUP_FILE
#include "samson/common/SamsonSetup.h"		// Own interface

namespace samson
{
    
	void createDirectory( std::string path )
	{
		if( mkdir(path.c_str()	, 0755) == -1 )
		{
			if( errno != EEXIST )
				LM_X(1,("Error creating directory %s", path.c_str()));
		}
	}
    
    
#pragma mark SetupItemCollection
    
    void  SetupItemCollection::add( std::string _name , std::string _default_value , std::string _description )
    {
        if( items.findInMap(_name) != NULL )
        {
            LM_W(("Item %s already added to setup... ignoring" , _name.c_str() ));
            return;
        }
        
        items.insertInMap( _name , new SetupItem( _name , _default_value , _description) );
        
    }
    
    
    void SetupItemCollection::load( std::string fileName )
    {
		FILE *file = fopen( fileName.c_str()  ,"r");
		if (!file)
		{
            LM_W(("Warning: Setup file %s not found\n" , fileName.c_str() ));
		}
		else
		{
			
			char line[2000];
			while( fgets(line, sizeof(line), file))
			{
				au::CommandLine c;
				c.parse(line);
				
				if( c.get_num_arguments() == 0 )
					continue;
				
				// Skip comments
				std::string mainCommand = c.get_argument(0);
				if( mainCommand[0] == '#' )
					continue;
				
				if (c.get_num_arguments() >= 2)	
				{
					std::string name = c.get_argument(0);
					std::string value =  c.get_argument(1);
                    
                    SetupItem*item = items.findInMap(name);
                    
                    if( item )
                        item->setValue(value);
                    else
                        LM_W(("Parameter %s found in setup file %s not included since it is not defined." , name.c_str() , fileName.c_str() ));
                    
				}
			}
            
			fclose(file);
		}        
    }

    std::string SetupItemCollection::getValueForParameter( std::string name )
    {
        SetupItem*item = items.findInMap(name);
        
        if( !item )
            LM_X(1, ("Parameter %s not defined in the setup. This is not acceptable", name.c_str()));
        
        return item->getValue();
        
    }
    
    bool SetupItemCollection::isParameterDefined( std::string name )
    {
        return ( items.findInMap(name) != NULL);
    }
    
    
    void SetupItemCollection::setValueForParameter( std::string name ,std::string value )
    {
        SetupItem*item = items.findInMap(name);
        
        if( !item )
        {
            LM_W(("Parameter %s not defined in the setup. This is not acceptable", name.c_str()));
            return;
        }
        
        item->setValue( value );
    }
    
    std::string SetupItemCollection::str()
    {
        std::ostringstream output;
        output << "Samson setup parameters:\n";
        output << "---------------------------------------------------------------------------------------------------\n";

        output << std::left;
        
        // Heder
        output << "\n";
        output << std::setw(40) << "Parameter" << " ";
        output << std::setw(20) << "Default value" << " ";
        output << std::setw(20) << "Value in setup.txt" << " ";
{'fraction_non_alphanumeric': 0.1634777715250422, 'fraction_numerical': 0.006471581316826111, 'mean_word_length': 1.7164692395873138, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7229433', 'n_tokens_mistral': 2393, 'n_tokens_neox': 2184, 'n_words': 464}