<commit_before>package MediaWords::StoryVectors;

# methods to generate the story_sentences and story_sentence_words and associated aggregated tables

use strict;

use Encode;
use Perl6::Say;

use Lingua::EN::Sentence::MediaWords;
use MediaWords::Util::Stemmer;
use MediaWords::Util::StopWords;

use Date::Format;
use Date::Parse;

# minimum length of words in story_sentence_words
use constant MIN_STEM_LENGTH => 3;

# if story is a ref, return itself, otherwise treat it as a stories_id and query for the story ref
sub _get_story
{
    my ( $db, $story ) = @_;

    if ( ref( $story ) )
    {
        return $story;
    }
    else
    {
        return $db->query( "select stories_id, publish_date, media_id from stories where stories_id = ?", $story )->hash;
    }
}

# get the text for the given story by concatenating all of the download_texts for the story
sub _get_story_text
{
    my ( $db, $stories_id ) = @_;

    my $download_texts = $db->query(
        "select dt.download_text from downloads d, download_texts dt " .
        "  where dt.downloads_id = d.downloads_id and d.stories_id = ? order by d.downloads_id",
        $stories_id )->hashes;
        
    return join( ". ", map { $_->{ download_text } } @{ $download_texts } );

}

# given a hash of word counts by sentence, insert the words into the db using sql COPY for performance
sub _insert_story_sentence_words
{
    my ( $db, $story, $word_counts ) = @_;<commit_msg>Update comment since sql COPY command is no longer used in _insert_story_sentence_words<commit_after>package MediaWords::StoryVectors;

# methods to generate the story_sentences and story_sentence_words and associated aggregated tables

use strict;

use Encode;
use Perl6::Say;

use Lingua::EN::Sentence::MediaWords;
use MediaWords::Util::Stemmer;
use MediaWords::Util::StopWords;

use Date::Format;
use Date::Parse;

# minimum length of words in story_sentence_words
use constant MIN_STEM_LENGTH => 3;

# if story is a ref, return itself, otherwise treat it as a stories_id and query for the story ref
sub _get_story
{
    my ( $db, $story ) = @_;

    if ( ref( $story ) )
    {
        return $story;
    }
    else
    {
        return $db->query( "select stories_id, publish_date, media_id from stories where stories_id = ?", $story )->hash;
    }
}

# get the text for the given story by concatenating all of the download_texts for the story
sub _get_story_text
{
    my ( $db, $stories_id ) = @_;

    my $download_texts = $db->query(
        "select dt.download_text from downloads d, download_texts dt " .
        "  where dt.downloads_id = d.downloads_id and d.stories_id = ? order by d.downloads_id",
        $stories_id )->hashes;
        
    return join( ". ", map { $_->{ download_text } } @{ $download_texts } );

}

# given a hash of word counts by sentence, insert the words into the db
sub _insert_story_sentence_words
{
    my ( $db, $story, $word_counts ) = @_;
{'fraction_non_alphanumeric': 0.10065046217048956, 'fraction_numerical': 0.0013693940431359123, 'mean_word_length': 3.5585023400936038, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4986677', 'n_tokens_mistral': 971, 'n_tokens_neox': 937, 'n_words': 310}