<commit_before>#!/usr/bin/env perl

# create media_tag_tag_counts table by querying the database tags / feeds / stories

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
    use lib "$FindBin::Bin/.";
}

use MediaWords::DB;
use Modern::Perl "2012";
use MediaWords::CommonLibs;

use DBIx::Simple::MediaWords;
use TableCreationUtils;
use Readonly;
use File::Temp qw/ tempfile tempdir /;
use Archive::Zip qw( :ERROR_CODES :CONSTANTS );
use File::Copy;
use List::Util qw(first max maxstr min minstr reduce shuffle sum);
use MediaWords::Controller::Dashboard;
use IO::Compress::Zip qw(:all);

use Getopt::Long;
use Date::Parse;
use Data::Dumper;
use Carp;
use Dir::Self;
use Cwd;

my $_stories_id_window_size = 1000;

# base dir
my $_base_dir = __DIR__ . '/..';

# if set by the --dashboards option, only dump stories in media sets belonging to the given dashboards
my $_dump_dashboards;

sub get_max_stories_id
{
    my ( $dbh ) = @_;

    my $max_stories_id_row = $dbh->query( "select max(stories_id) as max_id from story_sentence_words" );

    my $max_stories_id = $max_stories_id_row->hash()->{ max_id };

    return $max_stories_id;
}

sub get_min_stories_id
{
    my ( $dbh ) = @_;

    my $min_stories_id_row = $dbh->query( "select min(stories_id) as min_id from story_sentence_words" );

    my $min_stories_id = $min_stories_id_row->hash()->{ min_id };

    return $min_stories_id;
}

sub scroll_stories_id_window
{
    my ( $_stories_id_start, $_stories_id_stop, $max_stories_id ) = @_;

    $_stories_id_start = $_stories_id_stop + 1;
    $_stories_id_stop  = $_stories_id_start + $_stories_id_window_size - 1;

    $_stories_id_stop = min( $_stories_id_stop, $max_stories_id );

    return ( $_stories_id_start, $_stories_id_stop );
}

sub isNonnegativeInteger
{
    my ( $val ) = @_;

    return int( $val ) eq $val;
}

# return an in clause for a table identified by $table_name with a media_id that belongs
# to a media_set in one of the $_dump_dashboards dashboards.
# if $_dump_dashboards is empty, return an empty string.
sub get_dashboards_clause
{
    my ( $table_name ) = @_;

    return '' unless ( @{ $_dump_dashboards } );

    my $dashboards_list = join( ',', @{ $_dump_dashboards } );

    my $clause =
      "and ${ table_name }.media_id in " .
      "  ( select msmm.media_id from media_sets_media_map msmm, dashboard_media_sets dms " .
      "      where msmm.media_sets_id = dms.media_sets_id and dms.dashboards_id in ( $dashboards_list ) )";

    return $clause;
}

sub dump_story_words
{

    my ( $dbh, $dir, $first_dumped_id, $last_dumped_id ) = @_;

    if ( !defined( $first_dumped_id ) )
    {
        $first_dumped_id = 0;
    }

    if ( !defined( $last_dumped_id ) )
    {
        my $max_stories_id = get_max_stories_id( $dbh );
        $last_dumped_id = $max_stories_id;<commit_msg>Fix error in dump stpry tables script. Can't use an undefined value as an ARRAY reference at /space/mediacloud/mediacloud-RELEASE_20121126/script/mediawords_dump_story_tables.pl line 92.<commit_after>#!/usr/bin/env perl

# create media_tag_tag_counts table by querying the database tags / feeds / stories

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
    use lib "$FindBin::Bin/.";
}

use MediaWords::DB;
use Modern::Perl "2012";
use MediaWords::CommonLibs;

use DBIx::Simple::MediaWords;
use TableCreationUtils;
use Readonly;
use File::Temp qw/ tempfile tempdir /;
use Archive::Zip qw( :ERROR_CODES :CONSTANTS );
use File::Copy;
use List::Util qw(first max maxstr min minstr reduce shuffle sum);
use MediaWords::Controller::Dashboard;
use IO::Compress::Zip qw(:all);

use Getopt::Long;
use Date::Parse;
use Data::Dumper;
use Carp;
use Dir::Self;
use Cwd;

my $_stories_id_window_size = 1000;

# base dir
my $_base_dir = __DIR__ . '/..';

# if set by the --dashboards option, only dump stories in media sets belonging to the given dashboards
my $_dump_dashboards;

sub get_max_stories_id
{
    my ( $dbh ) = @_;

    my $max_stories_id_row = $dbh->query( "select max(stories_id) as max_id from story_sentence_words" );

    my $max_stories_id = $max_stories_id_row->hash()->{ max_id };

    return $max_stories_id;
}

sub get_min_stories_id
{
    my ( $dbh ) = @_;

    my $min_stories_id_row = $dbh->query( "select min(stories_id) as min_id from story_sentence_words" );

    my $min_stories_id = $min_stories_id_row->hash()->{ min_id };

    return $min_stories_id;
}

sub scroll_stories_id_window
{
    my ( $_stories_id_start, $_stories_id_stop, $max_stories_id ) = @_;

    $_stories_id_start = $_stories_id_stop + 1;
    $_stories_id_stop  = $_stories_id_start + $_stories_id_window_size - 1;

    $_stories_id_stop = min( $_stories_id_stop, $max_stories_id );

    return ( $_stories_id_start, $_stories_id_stop );
}

sub isNonnegativeInteger
{
    my ( $val ) = @_;

    return int( $val ) eq $val;
}

# return an in clause for a table identified by $table_name with a media_id that belongs
# to a media_set in one of the $_dump_dashboards dashboards.
# if $_dump_dashboards is empty, return an empty string.
sub get_dashboards_clause
{
    my ( $table_name ) = @_;

    return '' unless ( defined( $_dump_dashboards) && @{ $_dump_dashboards } );

    my $dashboards_list = join( ',', @{ $_dump_dashboards } );

    my $clause =
      "and ${ table_name }.media_id in " .
      "  ( select msmm.media_id from media_sets_media_map msmm, dashboard_media_sets dms " .
      "      where msmm.media_sets_id = dms.media_sets_id and dms.dashboards_id in ( $dashboards_list ) )";

    return $clause;
}

sub dump_story_words
{

    my ( $dbh, $dir, $first_dumped_id, $last_dumped_id ) = @_;

    if ( !defined( $first_dumped_id ) )
    {
        $first_dumped_id = 0;
    }

    if ( !defined( $last_dumped_id ) )
    {
        my $max_stories_id = get_max_stories_id( $dbh );
        $last_dumped_id = $max_stories_id;
{'fraction_non_alphanumeric': 0.11904761904761904, 'fraction_numerical': 0.005403579871664978, 'mean_word_length': 3.8628899835796386, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4109464', 'n_tokens_mistral': 2279, 'n_tokens_neox': 2188, 'n_words': 534}