<commit_before>#!/usr/bin/perl -w

# create media_tag_tag_counts table by querying the database tags / feeds / stories

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use MediaWords::DB;
use MediaWords::CommonLibs;

use DBIx::Simple::MediaWords;
use XML::LibXML;
use Getopt::Long;
use Readonly;
use Carp;
use MIME::Base64;
use Try::Tiny;
use MediaWords::Util::DatabaseRestore;

sub main
{

    my Readonly $usage =
      'USAGE: ./mediawords_dump_restore_table_sql.pl --table_name foo --sql_dump_file dump_file --line_number_file file ';

    my ( $table_name, $sql_dump_file, $line_number_file, $output_file );

    GetOptions(
        'table_name=s'       => \$table_name,
        'sql_dump_file=s'    => \$sql_dump_file,
        'line_number_file=s' => \$line_number_file,
        'output_file=s'      => \$output_file,
    ) or die "$usage\n";

    die "$usage\n"
      unless $table_name && $sql_dump_file && $line_number_file && $output_file;

    MediaWords::Util::DatabaseRestore::test_opening_files( $line_number_file, $sql_dump_file );

    open my $OUTPUT_FILE, '>', $output_file;

    #say STDERR Dumper( [ $table_name, $sql_dump_file, $line_number_file ] );

    say STDERR "starting --  " . localtime();

    my $start_and_end_lines =
      MediaWords::Util::DatabaseRestore::get_start_and_end_line_for_table( $line_number_file, $table_name );

    my $start_line = $start_and_end_lines->{ start_line };
    my $end_line   = $start_and_end_lines->{ end_line };
<commit_msg>Add display only option.<commit_after>#!/usr/bin/perl -w

# create media_tag_tag_counts table by querying the database tags / feeds / stories

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use MediaWords::DB;
use MediaWords::CommonLibs;

use DBIx::Simple::MediaWords;
use XML::LibXML;
use Getopt::Long;
use Readonly;
use Carp;
use MIME::Base64;
use Try::Tiny;
use MediaWords::Util::DatabaseRestore;

sub main
{

    my Readonly $usage =
      'USAGE: ./mediawords_dump_restore_table_sql.pl --table_name foo --sql_dump_file dump_file --line_number_file file ';

    my ( $table_name, $sql_dump_file, $line_number_file, $output_file, $display_only );

    $display_only = 0;

    GetOptions(
        'table_name=s'       => \$table_name,
        'sql_dump_file=s'    => \$sql_dump_file,
        'line_number_file=s' => \$line_number_file,
        'output_file=s'      => \$output_file,
        'display_line_numbers_only=b' => \$display_only,
    ) or die "$usage\n";

    die "$usage\n"
      unless $table_name && $sql_dump_file && $line_number_file && $output_file;

    MediaWords::Util::DatabaseRestore::test_opening_files( $line_number_file, $sql_dump_file );

    open my $OUTPUT_FILE, '>', $output_file;

    #say STDERR Dumper( [ $table_name, $sql_dump_file, $line_number_file ] );

    say STDERR "starting --  " . localtime();

    my $start_and_end_lines =
      MediaWords::Util::DatabaseRestore::get_start_and_end_line_for_table( $line_number_file, $table_name );

    my $start_line = $start_and_end_lines->{ start_line };
    my $end_line   = $start_and_end_lines->{ end_line };

    say STDERR "start line: $start_line, end line: $end_line ";

    if ( $display_only )
    {
	exit;
    }

{'fraction_non_alphanumeric': 0.13418725221103994, 'fraction_numerical': 0.0015248551387618177, 'mean_word_length': 3.816446402349486, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '257', 'n_tokens_mistral': 1233, 'n_tokens_neox': 1173, 'n_words': 237}