<commit_before>package MediaWords::Util::Config;
use Modern::Perl "2012";
use MediaWords::CommonLibs;

# Parse and return data from mediawords.yml config file.

# This code should be used instead of MediaWords->config in general, b/c
# MediaWords::Util::Config::get_config will work both from within and without
# catalyst (for instance in stand alone command line scripts).

# in the catalyst case, the core MediaWords script calls the set_config
# function to set the returned config object to the already generated
# config object for the app.

use strict;

use Carp;
use Dir::Self;
use Config::Any;

# cache config object so that it remains the same from call to call
my $_config;

# base dir
my $_base_dir = __DIR__ . '/../../..';

sub get_config
{

    if ( $_config )
    {
        return $_config;
    }

    # TODO: This should be standardized
    set_config_file( $_base_dir . '/mediawords.yml' );

    return $_config;
}

# set the cached config object given a file path
sub set_config_file
{
    my $config_file = shift;

    -r $config_file or croak "Can't read from $config_file";

    #print "config:file: $config_file\n";
    set_config( Config::Any->load_files( { files => [ $config_file ], use_ext => 1 } )->[ 0 ]->{ $config_file } );
}

# set the cached config object
sub set_config
{
    my ( $config ) = @_;

    if ( $_config )
    {
        carp( "config object already cached" );
    }

    $_config = set_defaults( $config );

    verify_settings( $_config );
}

sub set_defaults
{
    my ( $config ) = @_;

    $config->{ mediawords }->{ script_dir }                       ||= "$_base_dir/script";
    $config->{ mediawords }->{ data_dir }                         ||= "$_base_dir/data";
    $config->{ mediawords }->{ language }                         ||= "en_US_and_ru_RU";
    $config->{ mediawords }->{ user_agent }                       ||= "Mozilla/5.0 (Linux) MediaCloud/1.0";
    $config->{ mediawords }->{ owner }                            ||= "<EMAIL>";
    $config->{ mediawords }->{ always_show_stack_traces }         ||= "no";
    $config->{ mediawords }->{ disable_json_top_500_words_cache } ||= "no";
    $config->{ mediawords }->{ default_home_page }                ||= "admin/media/list";
    $config->{ mediawords }->{ recaptcha_public_key }             ||= "<KEY>";
    $config->{ mediawords }->{ recaptcha_private_key }            ||= "<KEY>";
    $config->{ mediawords }->{ download_storage_locations }       ||= [ 'tar' ];
    $config->{ mediawords }->{ read_tar_downloads_from_gridfs }   ||= 'no';
    $config->{ mediawords }->{ read_file_downloads_from_gridfs }  ||= 'no';
    $config->{ mediawords }->{ extractor_method }                 ||= 'HeuristicExtractor';<commit_msg>change default storage to file<commit_after>package MediaWords::Util::Config;
use Modern::Perl "2012";
use MediaWords::CommonLibs;

# Parse and return data from mediawords.yml config file.

# This code should be used instead of MediaWords->config in general, b/c
# MediaWords::Util::Config::get_config will work both from within and without
# catalyst (for instance in stand alone command line scripts).

# in the catalyst case, the core MediaWords script calls the set_config
# function to set the returned config object to the already generated
# config object for the app.

use strict;

use Carp;
use Dir::Self;
use Config::Any;

# cache config object so that it remains the same from call to call
my $_config;

# base dir
my $_base_dir = __DIR__ . '/../../..';

sub get_config
{

    if ( $_config )
    {
        return $_config;
    }

    # TODO: This should be standardized
    set_config_file( $_base_dir . '/mediawords.yml' );

    return $_config;
}

# set the cached config object given a file path
sub set_config_file
{
    my $config_file = shift;

    -r $config_file or croak "Can't read from $config_file";

    #print "config:file: $config_file\n";
    set_config( Config::Any->load_files( { files => [ $config_file ], use_ext => 1 } )->[ 0 ]->{ $config_file } );
}

# set the cached config object
sub set_config
{
    my ( $config ) = @_;

    if ( $_config )
    {
        carp( "config object already cached" );
    }

    $_config = set_defaults( $config );

    verify_settings( $_config );
}

sub set_defaults
{
    my ( $config ) = @_;

    $config->{ mediawords }->{ script_dir }                       ||= "$_base_dir/script";
    $config->{ mediawords }->{ data_dir }                         ||= "$_base_dir/data";
    $config->{ mediawords }->{ language }                         ||= "en_US_and_ru_RU";
    $config->{ mediawords }->{ user_agent }                       ||= "Mozilla/5.0 (Linux) MediaCloud/1.0";
    $config->{ mediawords }->{ owner }                            ||= "<EMAIL>";
    $config->{ mediawords }->{ always_show_stack_traces }         ||= "no";
    $config->{ mediawords }->{ disable_json_top_500_words_cache } ||= "no";
    $config->{ mediawords }->{ default_home_page }                ||= "admin/media/list";
    $config->{ mediawords }->{ recaptcha_public_key }             ||= "<KEY>";
    $config->{ mediawords }->{ recaptcha_private_key }            ||= "<KEY>";
    $config->{ mediawords }->{ download_storage_locations }       ||= [ 'file' ];
    $config->{ mediawords }->{ read_tar_downloads_from_gridfs }   ||= 'no';
    $config->{ mediawords }->{ read_file_downloads_from_gridfs }  ||= 'no';
    $config->{ mediawords }->{ extractor_method }                 ||= 'HeuristicExtractor';
{'fraction_non_alphanumeric': 0.1476608187134503, 'fraction_numerical': 0.004751461988304093, 'mean_word_length': 2.695476029709656, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5298484', 'n_tokens_mistral': 1769, 'n_tokens_neox': 1690, 'n_words': 463}