<commit_before>package FixMyStreet::Script::Reports;

use strict;
use warnings;

use CronFns;
use DateTime::Format::Pg;

use Utils;
use Utils::OpenStreetMap;
use mySociety::MaPit;

use FixMyStreet;
use FixMyStreet::Cobrand;
use FixMyStreet::DB;
use FixMyStreet::Email;
use FixMyStreet::SendReport;

sub send(;$) {
    my ($site_override) = @_;
    my $rs = FixMyStreet::DB->resultset('Problem');

    # Set up site, language etc.
    my ($verbose, $nomail, $debug_mode) = CronFns::options();

    my $base_url = FixMyStreet->config('BASE_URL');
    my $site = $site_override || CronFns::site($base_url);

    my $states = [ 'confirmed', 'fixed' ];
    $states = [ 'unconfirmed', 'confirmed', 'in progress', 'planned', 'closed', 'investigating' ] if $site eq 'zurich';
    my $unsent = $rs->search( {
        state => $states,
        whensent => undef,
        bodies_str => { '!=', undef },
    } );
    my (%notgot, %note);

    my $send_report = FixMyStreet::SendReport->new();
    my $senders = $send_report->get_senders;

    my $debug_unsent_count = 0;
    debug_print("starting to loop through unsent problem reports...") if $debug_mode;
    while (my $row = $unsent->next) {

        my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($row->cobrand)->new();

        if ($debug_mode) {
            $debug_unsent_count++;
            print "\n";
            debug_print("state=" . $row->state . ", bodies_str=" . $row->bodies_str . ($row->cobrand? ", cobrand=" . $row->cobrand : ""), $row->id);
        }

        # Cobranded and non-cobranded messages can share a database. In this case, the conf file
        # should specify a vhost to send the reports for each cobrand, so that they don't get sent
        # more than once if there are multiple vhosts running off the same database. The email_host
        # call checks if this is the host that sends mail for this cobrand.
        if (! $cobrand->email_host()) {
            debug_print("skipping because this host does not send reports for cobrand " . $cobrand->moniker, $row->id) if $debug_mode;
            next;
        }

        $cobrand->set_lang_and_domain($row->lang, 1);
        if ( $row->is_from_abuser) {
            $row->update( { state => 'hidden' } );
            debug_print("hiding because its sender is flagged as an abuser", $row->id) if $debug_mode;
            next;
        } elsif ( $row->title =~ /app store test/i ) {
            $row->update( { state => 'hidden' } );
            debug_print("hiding because it is an app store test message", $row->id) if $debug_mode;
            next;
        }

        # Template variables for the email
        my $email_base_url = $cobrand->base_url_for_report($row);
        my %h = map { $_ => $row->$_ } qw/id title detail name category latitude longitude used_map/;
        map { $h{$_} = $row->user->$_ || '' } qw/email phone/;
        $h{confirmed} = DateTime::Format::Pg->format_datetime( $row->confirmed->truncate (to => 'second' ) )
            if $row->confirmed;

        $h{query} = $row->postcode;
        $h{url} = $email_base_url . $row->url;
        $h{admin_url} = $row->admin_url($cobrand);
        $h{phone_line} = $h{phone} ? _('Phone:') . " $h{phone}\n\n" : '';
        if ($row->photo) {
            $h{has_photo} = _("This web page also contains a photo of the problem, provided by the user.") . "\n\n";
            $h{image_url} = $email_base_url . '/photo/' . $row->id . '.full.jpeg';
        } else {
            $h{has_photo} = '';
            $h{image_url} = '';
        }
        $h{fuzzy} = $row->used_map ? _('To view a map of the precise location of this issue')
            : _('The user could not locate the problem on a map, but to see the area around the location they entered');
        $h{closest_address} = '';

        $h{osm_url} = Utils::OpenStreetMap::short_url($h{latitude}, $h{longitude});
        if ( $row->used_map ) {
            $h{closest_address} = $cobrand->find_closest( $h{latitude}, $h{longitude}, $row );
            $h{osm_url} .= '?m';
        }

        if ( $cobrand->allow_anonymous_reports &&
             $row->user->email eq $cobrand->anonymous_account->{'email'}
         ) {
            $h{anonymous_report} = 1;
            $h{user_details} = _('This report was submitted anonymously');
        } else {
            $h{user_details} = sprintf(_('Name: %s'), $row->name) . "\n\n";
            $h{user_details} .= sprintf(_('Email: %s'), $row->user->email) . "\n\n";
        }

        $h{easting_northing} = '';

        if ($cobrand->can('process_additional_metadata_for_email')) {
            $cobrand->process_additional_metadata_for_email($row, \%h);
        }

        my $bodies = FixMyStreet::DB->resultset('Body')->search(
            { id => $row->bodies_str_ids },
            { order_by => 'name' },
        );

        my $missing;
        if ($row->bodies_missing) {
            my @missing = FixMyStreet::DB->resultset("Body")->search(
                { id => [ split /,/, $row->bodies_missing ] },
                { order_by => 'name' }
            )->get_column('name')->all;
            $missing = join(' / ', @missing) if @missing;
        }

        my @dear;
        my %reporters = ();
        my $skip = 0;
        while (my $body = $bodies->next) {
            my $sender_info = $cobrand->get_body_sender( $body, $row->category );
            my $sender = "FixMyStreet::SendReport::" . $sender_info->{method};

            if ( ! exists $senders->{ $sender } ) {
                warn sprintf "No such sender [ $sender ] for body %s ( %d )", $body->name, $body->id;
                next;
            }
            $reporters{ $sender } ||= $sender->new();

            if ( $reporters{ $sender }->should_skip( $row ) ) {
                $skip = 1;
                debug_print("skipped by sender " . $sender_info->{method} . " (might be due to previous failed attempts?)", $row->id) if $debug_mode;
            } else {
                debug_print("OK, adding recipient body " . $body->id . ":" . $body->name . ", " . $body->send_method, $row->id) if $debug_mode;
                push @dear, $body->name;
                $reporters{ $sender }->add_body( $body, $sender_info->{config} );
            }

            # If we are in the UK include eastings and northings, and nearest stuff
            if ( $cobrand->country eq 'GB' && !$h{easting} ) {
                my $coordsyst = 'G';
                my $first_area = $body->body_areas->first->area_id;
                my $area_info = mySociety::MaPit::call('area', $first_area);
                $coordsyst = 'I' if $area_info->{type} eq 'LGD';

                ( $h{easting}, $h{northing} ) = Utils::convert_latlon_to_en( $h{latitude}, $h{longitude}, $coordsyst );

                # email templates don't have conditionals so we need to format this here
                $h{easting_northing} = "Easting/Northing";
                $h{easting_northing} .= " (IE)" if $coordsyst eq 'I';<commit_msg>Display used send method in debug line.<commit_after>package FixMyStreet::Script::Reports;

use strict;
use warnings;

use CronFns;
use DateTime::Format::Pg;

use Utils;
use Utils::OpenStreetMap;
use mySociety::MaPit;

use FixMyStreet;
use FixMyStreet::Cobrand;
use FixMyStreet::DB;
use FixMyStreet::Email;
use FixMyStreet::SendReport;

sub send(;$) {
    my ($site_override) = @_;
    my $rs = FixMyStreet::DB->resultset('Problem');

    # Set up site, language etc.
    my ($verbose, $nomail, $debug_mode) = CronFns::options();

    my $base_url = FixMyStreet->config('BASE_URL');
    my $site = $site_override || CronFns::site($base_url);

    my $states = [ 'confirmed', 'fixed' ];
    $states = [ 'unconfirmed', 'confirmed', 'in progress', 'planned', 'closed', 'investigating' ] if $site eq 'zurich';
    my $unsent = $rs->search( {
        state => $states,
        whensent => undef,
        bodies_str => { '!=', undef },
    } );
    my (%notgot, %note);

    my $send_report = FixMyStreet::SendReport->new();
    my $senders = $send_report->get_senders;

    my $debug_unsent_count = 0;
    debug_print("starting to loop through unsent problem reports...") if $debug_mode;
    while (my $row = $unsent->next) {

        my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($row->cobrand)->new();

        if ($debug_mode) {
            $debug_unsent_count++;
            print "\n";
            debug_print("state=" . $row->state . ", bodies_str=" . $row->bodies_str . ($row->cobrand? ", cobrand=" . $row->cobrand : ""), $row->id);
        }

        # Cobranded and non-cobranded messages can share a database. In this case, the conf file
        # should specify a vhost to send the reports for each cobrand, so that they don't get sent
        # more than once if there are multiple vhosts running off the same database. The email_host
        # call checks if this is the host that sends mail for this cobrand.
        if (! $cobrand->email_host()) {
            debug_print("skipping because this host does not send reports for cobrand " . $cobrand->moniker, $row->id) if $debug_mode;
            next;
        }

        $cobrand->set_lang_and_domain($row->lang, 1);
        if ( $row->is_from_abuser) {
            $row->update( { state => 'hidden' } );
            debug_print("hiding because its sender is flagged as an abuser", $row->id) if $debug_mode;
            next;
        } elsif ( $row->title =~ /app store test/i ) {
            $row->update( { state => 'hidden' } );
            debug_print("hiding because it is an app store test message", $row->id) if $debug_mode;
            next;
        }

        # Template variables for the email
        my $email_base_url = $cobrand->base_url_for_report($row);
        my %h = map { $_ => $row->$_ } qw/id title detail name category latitude longitude used_map/;
        map { $h{$_} = $row->user->$_ || '' } qw/email phone/;
        $h{confirmed} = DateTime::Format::Pg->format_datetime( $row->confirmed->truncate (to => 'second' ) )
            if $row->confirmed;

        $h{query} = $row->postcode;
        $h{url} = $email_base_url . $row->url;
        $h{admin_url} = $row->admin_url($cobrand);
        $h{phone_line} = $h{phone} ? _('Phone:') . " $h{phone}\n\n" : '';
        if ($row->photo) {
            $h{has_photo} = _("This web page also contains a photo of the problem, provided by the user.") . "\n\n";
            $h{image_url} = $email_base_url . '/photo/' . $row->id . '.full.jpeg';
        } else {
            $h{has_photo} = '';
            $h{image_url} = '';
        }
        $h{fuzzy} = $row->used_map ? _('To view a map of the precise location of this issue')
            : _('The user could not locate the problem on a map, but to see the area around the location they entered');
        $h{closest_address} = '';

        $h{osm_url} = Utils::OpenStreetMap::short_url($h{latitude}, $h{longitude});
        if ( $row->used_map ) {
            $h{closest_address} = $cobrand->find_closest( $h{latitude}, $h{longitude}, $row );
            $h{osm_url} .= '?m';
        }

        if ( $cobrand->allow_anonymous_reports &&
             $row->user->email eq $cobrand->anonymous_account->{'email'}
         ) {
            $h{anonymous_report} = 1;
            $h{user_details} = _('This report was submitted anonymously');
        } else {
            $h{user_details} = sprintf(_('Name: %s'), $row->name) . "\n\n";
            $h{user_details} .= sprintf(_('Email: %s'), $row->user->email) . "\n\n";
        }

        $h{easting_northing} = '';

        if ($cobrand->can('process_additional_metadata_for_email')) {
            $cobrand->process_additional_metadata_for_email($row, \%h);
        }

        my $bodies = FixMyStreet::DB->resultset('Body')->search(
            { id => $row->bodies_str_ids },
            { order_by => 'name' },
        );

        my $missing;
        if ($row->bodies_missing) {
            my @missing = FixMyStreet::DB->resultset("Body")->search(
                { id => [ split /,/, $row->bodies_missing ] },
                { order_by => 'name' }
            )->get_column('name')->all;
            $missing = join(' / ', @missing) if @missing;
        }

        my @dear;
        my %reporters = ();
        my $skip = 0;
        while (my $body = $bodies->next) {
            my $sender_info = $cobrand->get_body_sender( $body, $row->category );
            my $sender = "FixMyStreet::SendReport::" . $sender_info->{method};

            if ( ! exists $senders->{ $sender } ) {
                warn sprintf "No such sender [ $sender ] for body %s ( %d )", $body->name, $body->id;
                next;
            }
            $reporters{ $sender } ||= $sender->new();

            if ( $reporters{ $sender }->should_skip( $row ) ) {
                $skip = 1;
                debug_print("skipped by sender " . $sender_info->{method} . " (might be due to previous failed attempts?)", $row->id) if $debug_mode;
            } else {
                debug_print("OK, adding recipient body " . $body->id . ":" . $body->name . ", " . $sender_info->{method}, $row->id) if $debug_mode;
                push @dear, $body->name;
                $reporters{ $sender }->add_body( $body, $sender_info->{config} );
            }

            # If we are in the UK include eastings and northings, and nearest stuff
            if ( $cobrand->country eq 'GB' && !$h{easting} ) {
                my $coordsyst = 'G';
                my $first_area = $body->body_areas->first->area_id;
                my $area_info = mySociety::MaPit::call('area', $first_area);
                $coordsyst = 'I' if $area_info->{type} eq 'LGD';

                ( $h{easting}, $h{northing} ) = Utils::convert_latlon_to_en( $h{latitude}, $h{longitude}, $coordsyst );

                # email templates don't have conditionals so we need to format this here
                $h{easting_northing} = "Easting/Northing";
                $h{easting_northing} .= " (IE)" if $coordsyst eq 'I';
{'fraction_non_alphanumeric': 0.16227630637079457, 'fraction_numerical': 0.0007158196134574087, 'mean_word_length': 2.2581623134328357, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 191, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '114031', 'n_tokens_mistral': 4686, 'n_tokens_neox': 4457, 'n_words': 1089}