<commit_before>package MediaWords::CM::GuessDate;

# guess the date of a spidered story using a combination of the story url, html, and
# a first guess date

# FIXME EST to GMT

use strict;
use warnings;

use MediaWords::CM::GuessDate::Result;

use DateTime;
use Date::Parse;
use HTML::TreeBuilder::LibXML;
use LWP::Simple;
use Regexp::Common qw(time);
use Date::Parse;
use List::Util qw(max min);

use MediaWords::CommonLibs;
use MediaWords::CM::GuessDate;
use MediaWords::DB;

# threshold of number of days a guess date can be off from the existing
# story date without dropping the guess
use constant DATE_GUESS_THRESHOLD => 14;

# Default hour to use when no time is present (minutes and seconds are going to both be 0)
# (12:00:00 because it looks nice and more or less fits within the same day in both California and Moscow)
use constant DEFAULT_HOUR => 12;

# only use the date from these guessing functions if the date is within DATE_GUESS_THRESHOLD days
# of the existing date for the story
my $_date_guess_functions = [
    {
        name     => 'guess_by_dc_date_issued',
        function => \&_guess_by_dc_date_issued
    },
    {
        name     => 'guess_by_dc_created',
        function => \&_guess_by_dc_created
    },
    {
        name     => 'guess_by_meta_pubdate',
        function => \&_guess_by_meta_pubdate
    },
    {
        name     => 'guess_by_meta_publish_date',
        function => \&_guess_by_meta_publish_date
    },
    {
        name     => 'guess_by_meta_item_publish_date',
        function => \&_guess_by_meta_item_publish_date
    },
    {
        name     => 'guess_by_og_article_published_time',
        function => \&_guess_by_og_article_published_time
    },
    {
        name     => 'guess_by_sailthru_date',
        function => \&_guess_by_sailthru_date
    },
    {
        name     => 'guess_by_abbr_published_updated_date',
        function => \&_guess_by_abbr_published_updated_date
    },
    {
        name     => 'guess_by_storydate',
        function => \&_guess_by_storydate
    },
    {
        name     => 'guess_by_datatime',
        function => \&_guess_by_datatime
    },
    {
        name     => 'guess_by_datetime_pubdate',
        function => \&_guess_by_datetime_pubdate
    },
    {
        name     => 'guess_by_url_and_date_text',
        function => \&_guess_by_url_and_date_text
    },
    {
        name     => 'guess_by_url',
        function => \&_guess_by_url
    },
    {
        name     => 'guess_by_class_date',
        function => \&_guess_by_class_date
    },
    {
        name     => 'guess_by_date_text',
        function => \&_guess_by_date_text
    },
    {
        name     => 'guess_by_existing_story_date',
        function => \&_guess_by_existing_story_date
    },
];

# return the first in a list of nodes matching the xpath pattern
sub _find_first_node
{
    my ( $html_tree, $xpath ) = @_;

    my @nodes = $html_tree->findnodes( $xpath );

    my $node = pop @nodes;

    return $node;
}

# get HTML::TreeBuilder::LibXML object representing the html
sub _get_html_tree
{
    my ( $html ) = @_;
<commit_msg>shift() gets the first value, not pop()<commit_after>package MediaWords::CM::GuessDate;

# guess the date of a spidered story using a combination of the story url, html, and
# a first guess date

# FIXME EST to GMT

use strict;
use warnings;

use MediaWords::CM::GuessDate::Result;

use DateTime;
use Date::Parse;
use HTML::TreeBuilder::LibXML;
use LWP::Simple;
use Regexp::Common qw(time);
use Date::Parse;
use List::Util qw(max min);

use MediaWords::CommonLibs;
use MediaWords::CM::GuessDate;
use MediaWords::DB;

# threshold of number of days a guess date can be off from the existing
# story date without dropping the guess
use constant DATE_GUESS_THRESHOLD => 14;

# Default hour to use when no time is present (minutes and seconds are going to both be 0)
# (12:00:00 because it looks nice and more or less fits within the same day in both California and Moscow)
use constant DEFAULT_HOUR => 12;

# only use the date from these guessing functions if the date is within DATE_GUESS_THRESHOLD days
# of the existing date for the story
my $_date_guess_functions = [
    {
        name     => 'guess_by_dc_date_issued',
        function => \&_guess_by_dc_date_issued
    },
    {
        name     => 'guess_by_dc_created',
        function => \&_guess_by_dc_created
    },
    {
        name     => 'guess_by_meta_pubdate',
        function => \&_guess_by_meta_pubdate
    },
    {
        name     => 'guess_by_meta_publish_date',
        function => \&_guess_by_meta_publish_date
    },
    {
        name     => 'guess_by_meta_item_publish_date',
        function => \&_guess_by_meta_item_publish_date
    },
    {
        name     => 'guess_by_og_article_published_time',
        function => \&_guess_by_og_article_published_time
    },
    {
        name     => 'guess_by_sailthru_date',
        function => \&_guess_by_sailthru_date
    },
    {
        name     => 'guess_by_abbr_published_updated_date',
        function => \&_guess_by_abbr_published_updated_date
    },
    {
        name     => 'guess_by_storydate',
        function => \&_guess_by_storydate
    },
    {
        name     => 'guess_by_datatime',
        function => \&_guess_by_datatime
    },
    {
        name     => 'guess_by_datetime_pubdate',
        function => \&_guess_by_datetime_pubdate
    },
    {
        name     => 'guess_by_url_and_date_text',
        function => \&_guess_by_url_and_date_text
    },
    {
        name     => 'guess_by_url',
        function => \&_guess_by_url
    },
    {
        name     => 'guess_by_class_date',
        function => \&_guess_by_class_date
    },
    {
        name     => 'guess_by_date_text',
        function => \&_guess_by_date_text
    },
    {
        name     => 'guess_by_existing_story_date',
        function => \&_guess_by_existing_story_date
    },
];

# return the first in a list of nodes matching the xpath pattern
sub _find_first_node
{
    my ( $html_tree, $xpath ) = @_;

    my @nodes = $html_tree->findnodes( $xpath );

    my $node = shift @nodes;

    return $node;
}

# get HTML::TreeBuilder::LibXML object representing the html
sub _get_html_tree
{
    my ( $html ) = @_;

{'fraction_non_alphanumeric': 0.10064205457463885, 'fraction_numerical': 0.003531300160513644, 'mean_word_length': 2.7023172905525845, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3197089', 'n_tokens_mistral': 2162, 'n_tokens_neox': 2109, 'n_words': 496}