<commit_before>package MediaWords::Solr;

use strict;

use Modern::Perl "2013";
use MediaWords::CommonLibs;

# functions for searching the solr server

use JSON;
use List::Util;
use Time::HiRes qw(gettimeofday tv_interval);

use MediaWords::DB;
use MediaWords::DBI::Stories;
use MediaWords::Languages::Language;
use MediaWords::Solr::PseudoQueries;
use MediaWords::Util::Config;
use MediaWords::Util::Text;
use MediaWords::Util::Web;
use List::MoreUtils qw ( uniq );
use HTTP::Request::Common qw( POST );

use Time::HiRes;

# numFound from last query() call, accessible get get_last_num_found
my $_last_num_found;

# mean number of sentences per story from the last search_stories() call
my $_last_sentences_per_story;

# get a solr url from the config, returning either the single
# url if there is one or a random member of the list if there is
# a list
sub get_solr_url
{
    my $urls = MediaWords::Util::Config::get_config->{ mediawords }->{ solr_url };

    my $i = int( Time::HiRes::time * 10000 ) % scalar( @{ $urls } );

    my $url = ref( $urls ) ? $urls->[ $i ] : $urls;

    $url =~ s~/+$~~;

    return $url;
}

# get the name of the currently live collection, as stored in database_variables in postgres
sub get_live_collection
{
    my ( $db ) = @_;

    my ( $collection ) = $db->query( "select value from database_variables where name = 'live_solr_collection' " )->flat;

    return $collection || 'collection1';
}

# get the name of the collection that is not get_live_collection()
sub get_staging_collection
{
    my ( $db ) = @_;

    my $live_collection = get_live_collection( $db );

    return $live_collection eq 'collection1' ? 'collection2' : 'collection1';
}

# swap which collection is live and which is staging
sub swap_live_collection
{
    my ( $db ) = @_;

    my $current_staging_collection = get_staging_collection( $db );

    $db->begin;

    $db->query( "delete from database_variables where name = 'live_solr_collection'" );
    $db->create( 'database_variables', { name => 'live_solr_collection', value => $current_staging_collection } );
    -

      $db->commit;
}

# get the numFound from the last solr query run
sub get_last_num_found
{
    return $_last_num_found;
}

# get the ratio of sentence per story for the last search_stories call
sub get_last_sentences_per_story
{
    return $_last_sentences_per_story;
}

sub _set_last_num_found
{
    my ( $res ) = @_;

    if ( defined( $res->{ response }->{ numFound } ) )
    {
        $_last_num_found = $res->{ response }->{ numFound };
    }
    elsif ( $res->{ grouped } )
    {
        my $group_key = ( keys( %{ $res->{ grouped } } ) )[ 0 ];

        $_last_num_found = $res->{ grouped }->{ $group_key }->{ matches };
    }
    else
    {
        $_last_num_found = undef;
    }

    print STDERR ( $_last_num_found ? $_last_num_found : 'undef' ) . " matches found.\n" if ( $ENV{ MC_SOLR_TRACE } );

}

# convert any and, or, or not operations in the argument to uppercase.  if the argument
# is a ref, call self on all elements of the list.
sub _uppercase_boolean_operators
{
    return unless ( $_[ 0 ] );

    if ( ref( $_[ 0 ] ) )
    {
        map { _uppercase_boolean_operators( $_ ) } @{ $_[ 0 ] };
    }
    else
    {
        $_[ 0 ] =~ s/\b(and|or|not)\b/uc($1)/ge;
    }
}

# execute a query on the solr server using the given params.
# return the raw encoded json from solr.  return a maximum of
# 1 million sentences.
sub query_encoded_json($$;$)
{
    my ( $db, $params, $c ) = @_;

    unless ( $params )
    {
        die 'params must be set.';
    }
    unless ( ref $params eq ref {} )
    {
        die 'params must be a hashref.';
    }

    $params->{ wt } = 'json';
    $params->{ rows } //= 1000;
    $params->{ df }   //= 'sentence';

    $params->{ rows } = List::Util::min( $params->{ rows }, 1000000 );

    _uppercase_boolean_operators( $params->{ q } );
    _uppercase_boolean_operators( $params->{ fq } );

    $params->{ q }  = MediaWords::Solr::PseudoQueries::transform_query( $params->{ q } );
    $params->{ fq } = MediaWords::Solr::PseudoQueries::transform_query( $params->{ fq } );

    # Ensure that only UTF-8 strings get passed to Solr
    my $encoded_params = MediaWords::Util::Text::recursively_encode_to_utf8( $params );

    my $url = sprintf( '%s/%s/select', get_solr_url(), get_live_collection( $db ) );

    my $ua = MediaWords::Util::Web::UserAgent;

    $ua->timeout( 300 );
    $ua->max_size( undef );

    if ( $ENV{ MC_SOLR_TRACE } )
    {
        say STDERR "Executing Solr query on $url ...";
        say STDERR 'Encoded parameters: ' . Dumper( $encoded_params );
    }

    my $t0 = [ gettimeofday ];

    my $request = POST( $url, $encoded_params );
    $request->content_type( 'application/x-www-form-urlencoded; charset=utf-8' );
    say STDERR "POST request: " . $request->as_string;

    my $res = $ua->request( $request );

    if ( $ENV{ MC_SOLR_TRACE } )
    {
        say STDERR "query returned in " . tv_interval( $t0, [ gettimeofday ] ) . "s.";
    }

    unless ( $res->is_success )
    {
        my $error_message;

        if ( MediaWords::Util::Web::response_error_is_client_side( $res ) )
        {

            # LWP error (LWP wasn't able to connect to the server or something like that)
            $error_message = 'LWP error: ' . $res->decoded_content;

        }
        else
        {

            my $status_code = $res->code;
            if ( $status_code =~ /^4\d\d$/ )
            {<commit_msg>Don't print out POST request<commit_after>package MediaWords::Solr;

use strict;

use Modern::Perl "2013";
use MediaWords::CommonLibs;

# functions for searching the solr server

use JSON;
use List::Util;
use Time::HiRes qw(gettimeofday tv_interval);

use MediaWords::DB;
use MediaWords::DBI::Stories;
use MediaWords::Languages::Language;
use MediaWords::Solr::PseudoQueries;
use MediaWords::Util::Config;
use MediaWords::Util::Text;
use MediaWords::Util::Web;
use List::MoreUtils qw ( uniq );
use HTTP::Request::Common qw( POST );

use Time::HiRes;

# numFound from last query() call, accessible get get_last_num_found
my $_last_num_found;

# mean number of sentences per story from the last search_stories() call
my $_last_sentences_per_story;

# get a solr url from the config, returning either the single
# url if there is one or a random member of the list if there is
# a list
sub get_solr_url
{
    my $urls = MediaWords::Util::Config::get_config->{ mediawords }->{ solr_url };

    my $i = int( Time::HiRes::time * 10000 ) % scalar( @{ $urls } );

    my $url = ref( $urls ) ? $urls->[ $i ] : $urls;

    $url =~ s~/+$~~;

    return $url;
}

# get the name of the currently live collection, as stored in database_variables in postgres
sub get_live_collection
{
    my ( $db ) = @_;

    my ( $collection ) = $db->query( "select value from database_variables where name = 'live_solr_collection' " )->flat;

    return $collection || 'collection1';
}

# get the name of the collection that is not get_live_collection()
sub get_staging_collection
{
    my ( $db ) = @_;

    my $live_collection = get_live_collection( $db );

    return $live_collection eq 'collection1' ? 'collection2' : 'collection1';
}

# swap which collection is live and which is staging
sub swap_live_collection
{
    my ( $db ) = @_;

    my $current_staging_collection = get_staging_collection( $db );

    $db->begin;

    $db->query( "delete from database_variables where name = 'live_solr_collection'" );
    $db->create( 'database_variables', { name => 'live_solr_collection', value => $current_staging_collection } );
    -

      $db->commit;
}

# get the numFound from the last solr query run
sub get_last_num_found
{
    return $_last_num_found;
}

# get the ratio of sentence per story for the last search_stories call
sub get_last_sentences_per_story
{
    return $_last_sentences_per_story;
}

sub _set_last_num_found
{
    my ( $res ) = @_;

    if ( defined( $res->{ response }->{ numFound } ) )
    {
        $_last_num_found = $res->{ response }->{ numFound };
    }
    elsif ( $res->{ grouped } )
    {
        my $group_key = ( keys( %{ $res->{ grouped } } ) )[ 0 ];

        $_last_num_found = $res->{ grouped }->{ $group_key }->{ matches };
    }
    else
    {
        $_last_num_found = undef;
    }

    print STDERR ( $_last_num_found ? $_last_num_found : 'undef' ) . " matches found.\n" if ( $ENV{ MC_SOLR_TRACE } );

}

# convert any and, or, or not operations in the argument to uppercase.  if the argument
# is a ref, call self on all elements of the list.
sub _uppercase_boolean_operators
{
    return unless ( $_[ 0 ] );

    if ( ref( $_[ 0 ] ) )
    {
        map { _uppercase_boolean_operators( $_ ) } @{ $_[ 0 ] };
    }
    else
    {
        $_[ 0 ] =~ s/\b(and|or|not)\b/uc($1)/ge;
    }
}

# execute a query on the solr server using the given params.
# return the raw encoded json from solr.  return a maximum of
# 1 million sentences.
sub query_encoded_json($$;$)
{
    my ( $db, $params, $c ) = @_;

    unless ( $params )
    {
        die 'params must be set.';
    }
    unless ( ref $params eq ref {} )
    {
        die 'params must be a hashref.';
    }

    $params->{ wt } = 'json';
    $params->{ rows } //= 1000;
    $params->{ df }   //= 'sentence';

    $params->{ rows } = List::Util::min( $params->{ rows }, 1000000 );

    _uppercase_boolean_operators( $params->{ q } );
    _uppercase_boolean_operators( $params->{ fq } );

    $params->{ q }  = MediaWords::Solr::PseudoQueries::transform_query( $params->{ q } );
    $params->{ fq } = MediaWords::Solr::PseudoQueries::transform_query( $params->{ fq } );

    # Ensure that only UTF-8 strings get passed to Solr
    my $encoded_params = MediaWords::Util::Text::recursively_encode_to_utf8( $params );

    my $url = sprintf( '%s/%s/select', get_solr_url(), get_live_collection( $db ) );

    my $ua = MediaWords::Util::Web::UserAgent;

    $ua->timeout( 300 );
    $ua->max_size( undef );

    if ( $ENV{ MC_SOLR_TRACE } )
    {
        say STDERR "Executing Solr query on $url ...";
        say STDERR 'Encoded parameters: ' . Dumper( $encoded_params );
    }

    my $t0 = [ gettimeofday ];

    my $request = POST( $url, $encoded_params );
    $request->content_type( 'application/x-www-form-urlencoded; charset=utf-8' );

    my $res = $ua->request( $request );

    if ( $ENV{ MC_SOLR_TRACE } )
    {
        say STDERR "query returned in " . tv_interval( $t0, [ gettimeofday ] ) . "s.";
    }

    unless ( $res->is_success )
    {
        my $error_message;

        if ( MediaWords::Util::Web::response_error_is_client_side( $res ) )
        {

            # LWP error (LWP wasn't able to connect to the server or something like that)
            $error_message = 'LWP error: ' . $res->decoded_content;

        }
        else
        {

            my $status_code = $res->code;
            if ( $status_code =~ /^4\d\d$/ )
            {
{'fraction_non_alphanumeric': 0.141507703595011, 'fraction_numerical': 0.007336757153338224, 'mean_word_length': 2.9857456140350878, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 80, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2321452', 'n_tokens_mistral': 3865, 'n_tokens_neox': 3670, 'n_words': 1013}