<commit_before>package FixMyStreet::App::Controller::Admin::ExorDefects;
use Moose;
use namespace::autoclean;

use Text::CSV;
use DateTime;
use mySociety::Random qw(random_bytes);

BEGIN { extends 'Catalyst::Controller'; }


sub begin : Private {
    my ( $self, $c ) = @_;

    $c->forward('/admin/begin');
}

sub index : Path : Args(0) {
    my ( $self, $c ) = @_;

    foreach (qw(error_message start_date end_date user_id)) {
        if ( defined $c->flash->{$_} ) {
            $c->stash->{$_} = $c->flash->{$_};
        }
    }

    my @inspectors = $c->cobrand->users->search({
        'user_body_permissions.permission_type' => 'report_inspect'
    }, {
            join => 'user_body_permissions',
            distinct => 1,
        }
    )->all;
    $c->stash->{inspectors} = \@inspectors;

    # Default start/end date is today
    my $now = DateTime->now( time_zone => 
        FixMyStreet->time_zone || FixMyStreet->local_time_zone );
    $c->stash->{start_date} ||= $now;
    $c->stash->{end_date} ||= $now;

}

sub download : Path('download') : Args(0) {
    my ( $self, $c ) = @_;

    if ( !$c->cobrand->can('exor_rdi_link_id') ) {
        # This only works on the Oxfordshire cobrand currently.
        $c->detach( '/page_error_404_not_found', [] );
    }

    my $parser = DateTime::Format::Strptime->new( pattern => '%d/%m/%Y' );
    my $start_date = $parser-> parse_datetime ( $c->get_param('start_date') );
    my $end_date = $parser-> parse_datetime ( $c->get_param('end_date') ) ;
    my $one_day = DateTime::Duration->new( days => 1 );

    my %params = (
        -and => [
            state => [ 'action scheduled' ],
            external_id => { '!=' => undef },
            'admin_log_entries.action' => 'inspected',
            'admin_log_entries.whenedited' => { '>=', $start_date },
            'admin_log_entries.whenedited' => { '<=', $end_date + $one_day },
        ]
    );

    my $user;
    if ( $c->get_param('user_id') ) {
        my $uid = $c->get_param('user_id');
        $params{'admin_log_entries.user_id'} = $uid;
        $user = $c->model('DB::User')->find( { id => $uid } );
    }

    my $problems = $c->cobrand->problems->search(
        \%params,
        {
            join => 'admin_log_entries',
            distinct => 1,
        }
    );

    if ( !$problems->count ) {
        if ( defined $user ) {
            $c->flash->{error_message} = _("No inspections by that inspector in the selected date range.");
        } else {
            $c->flash->{error_message} = _("No inspections in the selected date range.");
        }
        $c->flash->{start_date} = $start_date;
        $c->flash->{end_date} = $end_date;
        $c->flash->{user_id} = $user->id if $user;
        $c->res->redirect( $c->uri_for( '' ) );
    }

    # A single RDI file might contain inspections from multiple inspectors, so
    # we need to group inspections by inspector within G records.
    my $inspectors = {};
    my $inspector_initials = {};
    while ( my $report = $problems->next ) {
        my $user = $report->inspection_log_entry->user;
        $inspectors->{$user->id} ||= [];
        push @{ $inspectors->{$user->id} }, $report;
        unless ( $inspector_initials->{$user->id} ) {
            $inspector_initials->{$user->id} = $user->get_extra_metadata('initials');
        }
    }

    my $csv = Text::CSV->new({ binary => 1, eol => "" });

    my $p_count = 0;
    my $link_id = $c->cobrand->exor_rdi_link_id;

    # RDI first line is always the same
    $csv->combine("1", "1.8", "1.0.0.0", "ENHN", "");
    my @body = ($csv->string);

    my $i = 0;
    foreach my $inspector_id (keys %$inspectors) {
        my $inspections = $inspectors->{$inspector_id};
        my $initials = $inspector_initials->{$inspector_id};

        $csv->combine(
            "G", # start of an area/sequence
            $link_id, # area/link id, fixed value for our purposes
            "","", # must be empty
            $initials || "XX", # inspector initials
            $start_date->strftime("%y%m%d"), # date of inspection yymmdd
            "0700", # time of inspection hhmm, set to static value for now
            "D", # inspection variant, should always be D
            "INS", # inspection type, always INS
            "N", # Area of the county - north (N) or south (S)
            "", "", "", "" # empty fields
        );
        push @body, $csv->string;

        $csv->combine(
            "H", # initial inspection type
            "MC" # minor carriageway (changes depending on activity code)
        );
        push @body, $csv->string;

        foreach my $report (@$inspections) {
            my ($eastings, $northings) = $report->local_coords;
            my $location = "${eastings}E ${northings}N";
            my $description = sprintf("%s %s", $report->external_id || "", $report->get_extra_metadata('detailed_information') || "");
            my $activity_code = $report->defect_type ?
                $report->defect_type->get_extra_metadata('activity_code')
                : 'MC';
            my $traffic_information = $report->get_extra_metadata('traffic_information') ?
                'TM ' . $report->get_extra_metadata('traffic_information')
                : 'TM none';

            $csv->combine(
                "I", # beginning of defect record<commit_msg>[Oxfordshire] Include nearest info in RDI output.<commit_after>package FixMyStreet::App::Controller::Admin::ExorDefects;
use Moose;
use namespace::autoclean;

use Text::CSV;
use DateTime;
use mySociety::Random qw(random_bytes);

BEGIN { extends 'Catalyst::Controller'; }


sub begin : Private {
    my ( $self, $c ) = @_;

    $c->forward('/admin/begin');
}

sub index : Path : Args(0) {
    my ( $self, $c ) = @_;

    foreach (qw(error_message start_date end_date user_id)) {
        if ( defined $c->flash->{$_} ) {
            $c->stash->{$_} = $c->flash->{$_};
        }
    }

    my @inspectors = $c->cobrand->users->search({
        'user_body_permissions.permission_type' => 'report_inspect'
    }, {
            join => 'user_body_permissions',
            distinct => 1,
        }
    )->all;
    $c->stash->{inspectors} = \@inspectors;

    # Default start/end date is today
    my $now = DateTime->now( time_zone => 
        FixMyStreet->time_zone || FixMyStreet->local_time_zone );
    $c->stash->{start_date} ||= $now;
    $c->stash->{end_date} ||= $now;

}

sub download : Path('download') : Args(0) {
    my ( $self, $c ) = @_;

    if ( !$c->cobrand->can('exor_rdi_link_id') ) {
        # This only works on the Oxfordshire cobrand currently.
        $c->detach( '/page_error_404_not_found', [] );
    }

    my $parser = DateTime::Format::Strptime->new( pattern => '%d/%m/%Y' );
    my $start_date = $parser-> parse_datetime ( $c->get_param('start_date') );
    my $end_date = $parser-> parse_datetime ( $c->get_param('end_date') ) ;
    my $one_day = DateTime::Duration->new( days => 1 );

    my %params = (
        -and => [
            state => [ 'action scheduled' ],
            external_id => { '!=' => undef },
            'admin_log_entries.action' => 'inspected',
            'admin_log_entries.whenedited' => { '>=', $start_date },
            'admin_log_entries.whenedited' => { '<=', $end_date + $one_day },
        ]
    );

    my $user;
    if ( $c->get_param('user_id') ) {
        my $uid = $c->get_param('user_id');
        $params{'admin_log_entries.user_id'} = $uid;
        $user = $c->model('DB::User')->find( { id => $uid } );
    }

    my $problems = $c->cobrand->problems->search(
        \%params,
        {
            join => 'admin_log_entries',
            distinct => 1,
        }
    );

    if ( !$problems->count ) {
        if ( defined $user ) {
            $c->flash->{error_message} = _("No inspections by that inspector in the selected date range.");
        } else {
            $c->flash->{error_message} = _("No inspections in the selected date range.");
        }
        $c->flash->{start_date} = $start_date;
        $c->flash->{end_date} = $end_date;
        $c->flash->{user_id} = $user->id if $user;
        $c->res->redirect( $c->uri_for( '' ) );
    }

    # A single RDI file might contain inspections from multiple inspectors, so
    # we need to group inspections by inspector within G records.
    my $inspectors = {};
    my $inspector_initials = {};
    while ( my $report = $problems->next ) {
        my $user = $report->inspection_log_entry->user;
        $inspectors->{$user->id} ||= [];
        push @{ $inspectors->{$user->id} }, $report;
        unless ( $inspector_initials->{$user->id} ) {
            $inspector_initials->{$user->id} = $user->get_extra_metadata('initials');
        }
    }

    my $csv = Text::CSV->new({ binary => 1, eol => "" });

    my $p_count = 0;
    my $link_id = $c->cobrand->exor_rdi_link_id;

    # RDI first line is always the same
    $csv->combine("1", "1.8", "1.0.0.0", "ENHN", "");
    my @body = ($csv->string);

    my $i = 0;
    foreach my $inspector_id (keys %$inspectors) {
        my $inspections = $inspectors->{$inspector_id};
        my $initials = $inspector_initials->{$inspector_id};

        $csv->combine(
            "G", # start of an area/sequence
            $link_id, # area/link id, fixed value for our purposes
            "","", # must be empty
            $initials || "XX", # inspector initials
            $start_date->strftime("%y%m%d"), # date of inspection yymmdd
            "0700", # time of inspection hhmm, set to static value for now
            "D", # inspection variant, should always be D
            "INS", # inspection type, always INS
            "N", # Area of the county - north (N) or south (S)
            "", "", "", "" # empty fields
        );
        push @body, $csv->string;

        $csv->combine(
            "H", # initial inspection type
            "MC" # minor carriageway (changes depending on activity code)
        );
        push @body, $csv->string;

        foreach my $report (@$inspections) {
            my ($eastings, $northings) = $report->local_coords;

            my $location = "${eastings}E ${northings}N.";
            $location = "[DID NOT USE MAP] $location" unless $report->used_map;
            my $closest_address = $c->cobrand->find_closest($report, 1);
            if (%$closest_address) {
                $location .= " Nearest road: $closest_address->{road}." if $closest_address->{road};
                $location .= " Nearest postcode: $closest_address->{postcode}{postcode}." if $closest_address->{postcode};
            }

            my $description = sprintf("%s %s", $report->external_id || "", $report->get_extra_metadata('detailed_information') || "");
            my $activity_code = $report->defect_type ?
                $report->defect_type->get_extra_metadata('activity_code')
                : 'MC';
            my $traffic_information = $report->get_extra_metadata('traffic_information') ?
                'TM ' . $report->get_extra_metadata('traffic_information')
                : 'TM none';

            $csv->combine(
                "I", # beginning of defect record
{'fraction_non_alphanumeric': 0.17178079726446505, 'fraction_numerical': 0.004049311617025105, 'mean_word_length': 2.2940130409010075, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 216, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '133169', 'n_tokens_mistral': 3734, 'n_tokens_neox': 3581, 'n_words': 777}