<commit_before>package MediaWords::Controller::Search;

use Modern::Perl "2013";
use MediaWords::CommonLibs;

use strict;
use warnings;
use base 'Catalyst::Controller';

use MediaWords::Solr;
use MediaWords::Util::CSV;

=head1 NAME>

MediaWords::Controller::Health - Catalyst Controller

=head1 DESCRIPTION

Catalyst Controller for basic story search page

=cut

# get tag_sets_id for collection: tag set
sub _get_collection_tag_sets_id
{
    my ( $db ) = @_;

    my $tag_set = $db->query( "select * from tag_sets where name = 'collection'" )->hash
      || die( "Unable to find 'collection' tag set" );

    return $tag_set->{ tag_sets_id };
}

# list all collection tags, with media set names attached
sub tags : Local
{
    my ( $self, $c, $tag_sets_id ) = @_;

    my $db = $c->dbis;

    $tag_sets_id //= _get_collection_tag_sets_id( $db );

    my $tag_set = $db->find_by_id( 'tag_sets', $tag_sets_id ) || die( "Unable to find tag_set '$tag_sets_id'" );

    my $tags = $db->query( <<END, $tag_sets_id )->hashes;
with set_tags as (
    
    select t.* from tags t 
        where tag_sets_id = ? and
            tags_id not in ( select tags_id from media_sets where include_in_dump = false )
            
)
        
select t.*, ms.media_set_names, mtm.media_count
    from set_tags t
    
        left join ( 
            select count(*) media_count, mtm.tags_id 
                from media_tags_map mtm
                group by mtm.tags_id
        ) mtm on ( mtm.tags_id = t.tags_id )
        
        left join (
            select ms.tags_id,
                array_to_string( array_agg( d.name || ':' || ms.name ), '; ' ) media_set_names
            from media_sets ms
                join dashboard_media_sets dms on ( dms.media_sets_id = ms.media_sets_id )
                join dashboards d on ( d.dashboards_id = dms.dashboards_id ) 
            where ms.tags_id is not null
            group by ms.tags_id
        ) ms on ( t.tags_id = ms.tags_id )

    order by media_set_names, t.tags_id
END

    $c->stash->{ tags }     = $tags;
    $c->stash->{ tag_set }  = $tag_set;
    $c->stash->{ template } = 'search/tags.tt2';
}

# list all media sources associated with the given tag
sub media : Local
{
    my ( $self, $c, $tags_id ) = @_;

    die( "no tags_id" ) unless ( $tags_id );

    my $db = $c->dbis;

    my $media = $db->query( <<'END', $tags_id )->hashes;
select m.* 
    from media m join media_tags_map mtm on ( m.media_id = mtm.media_id )
    where mtm.tags_id = ?
    order by mtm.media_id
END

    $c->stash->{ media }    = $media;
    $c->stash->{ template } = 'search/media.tt2';
}

# search for stories using solr and return either a sampled list of stories in html or the full list in csv
sub index : Path : Args(0)
{
    my ( $self, $c ) = @_;

    my $q = $c->req->params->{ q } || '';
    my $l = $c->req->params->{ l };

    if ( !$q )
    {
        $c->stash->{ template } = 'search/search.tt2';
        $c->stash->{ title }    = 'Search';
        return;
    }

    my $db = $c->dbis;

    my $csv = $c->req->params->{ csv };

    my $solr_params = { q => $q };
    if ( !$csv )
    {
        $solr_params->{ sort } = 'random_1 asc';
        $solr_params->{ rows } = 100;<commit_msg>fix bug limiting csv download to 1000 stories<commit_after>package MediaWords::Controller::Search;

use Modern::Perl "2013";
use MediaWords::CommonLibs;

use strict;
use warnings;
use base 'Catalyst::Controller';

use MediaWords::Solr;
use MediaWords::Util::CSV;

=head1 NAME>

MediaWords::Controller::Health - Catalyst Controller

=head1 DESCRIPTION

Catalyst Controller for basic story search page

=cut

# get tag_sets_id for collection: tag set
sub _get_collection_tag_sets_id
{
    my ( $db ) = @_;

    my $tag_set = $db->query( "select * from tag_sets where name = 'collection'" )->hash
      || die( "Unable to find 'collection' tag set" );

    return $tag_set->{ tag_sets_id };
}

# list all collection tags, with media set names attached
sub tags : Local
{
    my ( $self, $c, $tag_sets_id ) = @_;

    my $db = $c->dbis;

    $tag_sets_id //= _get_collection_tag_sets_id( $db );

    my $tag_set = $db->find_by_id( 'tag_sets', $tag_sets_id ) || die( "Unable to find tag_set '$tag_sets_id'" );

    my $tags = $db->query( <<END, $tag_sets_id )->hashes;
with set_tags as (
    
    select t.* from tags t 
        where tag_sets_id = ? and
            tags_id not in ( select tags_id from media_sets where include_in_dump = false )
            
)
        
select t.*, ms.media_set_names, mtm.media_count
    from set_tags t
    
        left join ( 
            select count(*) media_count, mtm.tags_id 
                from media_tags_map mtm
                group by mtm.tags_id
        ) mtm on ( mtm.tags_id = t.tags_id )
        
        left join (
            select ms.tags_id,
                array_to_string( array_agg( d.name || ':' || ms.name ), '; ' ) media_set_names
            from media_sets ms
                join dashboard_media_sets dms on ( dms.media_sets_id = ms.media_sets_id )
                join dashboards d on ( d.dashboards_id = dms.dashboards_id ) 
            where ms.tags_id is not null
            group by ms.tags_id
        ) ms on ( t.tags_id = ms.tags_id )

    order by media_set_names, t.tags_id
END

    $c->stash->{ tags }     = $tags;
    $c->stash->{ tag_set }  = $tag_set;
    $c->stash->{ template } = 'search/tags.tt2';
}

# list all media sources associated with the given tag
sub media : Local
{
    my ( $self, $c, $tags_id ) = @_;

    die( "no tags_id" ) unless ( $tags_id );

    my $db = $c->dbis;

    my $media = $db->query( <<'END', $tags_id )->hashes;
select m.* 
    from media m join media_tags_map mtm on ( m.media_id = mtm.media_id )
    where mtm.tags_id = ?
    order by mtm.media_id
END

    $c->stash->{ media }    = $media;
    $c->stash->{ template } = 'search/media.tt2';
}

# search for stories using solr and return either a sampled list of stories in html or the full list in csv
sub index : Path : Args(0)
{
    my ( $self, $c ) = @_;

    my $q = $c->req->params->{ q } || '';
    my $l = $c->req->params->{ l };

    if ( !$q )
    {
        $c->stash->{ template } = 'search/search.tt2';
        $c->stash->{ title }    = 'Search';
        return;
    }

    my $db = $c->dbis;

    my $csv = $c->req->params->{ csv };

    my $solr_params = { q => $q };
    if ( $csv )
    {
        $solr_params->{ rows } = 100_000;
    }
    else
    {
        $solr_params->{ sort } = 'random_1 asc';
        $solr_params->{ rows } = 100;
{'fraction_non_alphanumeric': 0.1368646662584201, 'fraction_numerical': 0.005817513778322106, 'mean_word_length': 2.314561136478945, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 80, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6183202', 'n_tokens_mistral': 2410, 'n_tokens_neox': 2302, 'n_words': 638}