<commit_before>package MediaWords::Util::Config;

use strict;
use warnings;

use Modern::Perl "2015";
use MediaWords::CommonLibs;

# Parse and return data from mediawords.yml config file.

# This code should be used instead of MediaWords->config in general, b/c
# MediaWords::Util::Config::get_config will work both from within and without
# catalyst (for instance in stand alone command line scripts).

# in the catalyst case, the core MediaWords script calls the set_config
# function to set the returned config object to the already generated
# config object for the app.

use Config::Any;
use Dir::Self;
use Exporter 'import';
use Hash::Merge;

# cache config object so that it remains the same from call to call
my $_config;

# base dir
my $MC_ROOT_DIR;
my $_base_dir = __DIR__ . '/../../..';

our @EXPORT_OK = qw(get_config);

BEGIN
{
    use File::Basename;

    use File::Spec;

    my $file_dir = dirname( __FILE__ );

    use Cwd qw( realpath );

    my $source_rt = "$file_dir" . "/../../../";

    use File::Spec;

    $MC_ROOT_DIR = realpath( File::Spec->canonpath( $source_rt ) );
}

sub get_mc_root_dir
{
    return $MC_ROOT_DIR;
}

sub get_config
{

    if ( $_config )
    {
        return $_config;
    }

    # TODO: This should be standardized
    set_config_file( $_base_dir . '/mediawords.yml' );

    return $_config;
}

sub _parse_config_file
{
    my $config_file = shift;

    -r $config_file or MediaWords::CommonLibs::LOGCROAK( "Can't read from $config_file" );

    my $ret = Config::Any->load_files( { files => [ $config_file ], use_ext => 1 } )->[ 0 ]->{ $config_file };

    return $ret;
}
<commit_msg>Add helper to determine path to Media Cloud Python module root<commit_after>package MediaWords::Util::Config;

use strict;
use warnings;

use Modern::Perl "2015";
use MediaWords::CommonLibs;

# Parse and return data from mediawords.yml config file.

# This code should be used instead of MediaWords->config in general, b/c
# MediaWords::Util::Config::get_config will work both from within and without
# catalyst (for instance in stand alone command line scripts).

# in the catalyst case, the core MediaWords script calls the set_config
# function to set the returned config object to the already generated
# config object for the app.

use Config::Any;
use Dir::Self;
use Exporter 'import';
use Hash::Merge;

# cache config object so that it remains the same from call to call
my $_config;

# base dir
my $MC_ROOT_DIR;
my $_base_dir = __DIR__ . '/../../..';

our @EXPORT_OK = qw(get_config);

BEGIN
{
    use File::Basename;

    use File::Spec;

    my $file_dir = dirname( __FILE__ );

    use Cwd qw( realpath );

    my $source_rt = "$file_dir" . "/../../../";

    use File::Spec;

    $MC_ROOT_DIR = realpath( File::Spec->canonpath( $source_rt ) );
}

sub get_mc_root_dir
{
    return $MC_ROOT_DIR;
}

sub get_mc_python_dir()
{
    return get_mc_root_dir() . '/mediacloud/';
}

sub get_config
{

    if ( $_config )
    {
        return $_config;
    }

    # TODO: This should be standardized
    set_config_file( $_base_dir . '/mediawords.yml' );

    return $_config;
}

sub _parse_config_file
{
    my $config_file = shift;

    -r $config_file or MediaWords::CommonLibs::LOGCROAK( "Can't read from $config_file" );

    my $ret = Config::Any->load_files( { files => [ $config_file ], use_ext => 1 } )->[ 0 ]->{ $config_file };

    return $ret;
}

{'fraction_non_alphanumeric': 0.1237842617152962, 'fraction_numerical': 0.0035366931918656055, 'mean_word_length': 3.50730411686587, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '944085', 'n_tokens_mistral': 1168, 'n_tokens_neox': 1103, 'n_words': 363}