<commit_before>package Open311::PopulateServiceList;

use Moose;
use LWP::Simple;
use XML::Simple;
use FixMyStreet::App;
use Open311;

has council_list => ( is => 'ro' );
has found_contacts => ( is => 'rw', default => sub { [] } );

has _current_council => ( is => 'rw' );
has _current_open311 => ( is => 'rw' );
has _current_service => ( is => 'rw' );

my $council_list = FixMyStreet::App->model('DB::Open311conf');

sub process_councils {
    my $self = shift;

    while ( my $council = $self->council_list->next ) {
        next unless $council->endpoint;
        $self->_current_council( $council );
        $self->process_council;
    }
}

sub process_council {
    my $self = shift;

    my $open311 = Open311->new(
        endpoint => $self->_current_council->endpoint,
        jurisdiction => $self->_current_council->jurisdiction,
        api_key => $self->_current_council->api_key
    );

    $self->_current_open311( $open311 );
    $self->_check_endpoints;

    my $list = $open311->get_service_list;
    unless ( $list ) {
        warn "ERROR: no service list found for " . $self->_current_council->area_id . "\n";
        return;
    }
    $self->process_services( $list );
}



sub _check_endpoints {
    my $self = shift;

    # west berks end point not standard
    if ( $self->_current_council->area_id == 2619 ) {
        $self->_current_open311->endpoints(
            {
                services => 'Services',
                requests => 'Requests'
            }
        );
    }
}


sub process_services {
    my $self = shift;
    my $list = shift;

    $self->found_contacts( [] );
    foreach my $service ( @{ $list->{service} } ) {
        $self->_current_service( $service );
        $self->process_service;
    }
    $self->_delete_contacts_not_in_service_list;
}

sub process_service {
    my $self = shift;

    my $category = $self->_current_council->area_id == 2218 ? 
                    $self->_current_service->{description} : 
                    $self->_current_service->{service_name};

    print $self->_current_service->{service_code} . ': ' . $category .  "\n";
    my $contacts = FixMyStreet::App->model( 'DB::Contact')->search(
        {
            area_id => $self->_current_council->area_id,
            -OR => [
                email => $self->_current_service->{service_code},
                category => $category,
            ]
        }
    );

    if ( $contacts->count() > 1 ) {
        printf(
            "Multiple contacts for service code %s, category %s - Skipping\n",
            $self->_current_service->{service_code},
            $category,
        );

        # best to not mark them as deleted as we don't know what we're doing
        while ( my $contact = $contacts->next ) {
            push @{ $self->found_contacts }, $contact->email;
        }

        return;
    }

    my $contact = $contacts->first;

    if ( $contact ) {
        $self->_handle_existing_contact( $contact );
    } else {
        $self->_create_contact;
    }
}

sub _handle_existing_contact {
    my ( $self, $contact ) = @_;

    my $service_name = $self->_normalize_service_name;

    print $self->_current_council->area_id . " already has a contact for service code " . $self->_current_service->{service_code} . "\n";

    if ( $contact->deleted || $service_name ne $contact->category || $self->_current_service->{service_code} ne $contact->email ) {
        eval {
            $contact->update(
                {
                    category => $service_name,
                    email => $self->_current_service->{service_code},
                    confirmed => 1,
                    deleted => 0,
                    editor => $0,
                    whenedited => \'ms_current_timestamp()',
                    note => 'automatically undeleted by script',
                }
            );
        };

        if ( $@ ) {
            warn "Failed to update contact for service code " . $self->_current_service->{service_code} . " for council @{[$self->_current_council->area_id]}: $@\n";
            return;
        }
    }

    push @{ $self->found_contacts }, $self->_current_service->{service_code};
}

sub _create_contact {
    my $self = shift;

    my $service_name = $self->_normalize_service_name;

    my $contact;
    eval {
        $contact = FixMyStreet::App->model( 'DB::Contact')->create(
            {
                email => $self->_current_service->{service_code},
                area_id => $self->_current_council->area_id,
                category => $service_name,
                confirmed => 1,
                deleted => 0,
                editor => $0,
                whenedited => \'ms_current_timestamp()',
                note => 'created automatically by script',
            }
        );
    };

    if ( $@ ) {
        warn "Failed to create contact for service code " . $self->_current_service->{service_code} . " for council @{[$self->_current_council->area_id]}: $@\n";
        return;
    }

    if ( $contact and lc( $self->_current_service->{metadata} ) eq 'true' ) {
        $self->_add_meta_to_contact( $contact );
    }

    if ( $contact ) {
        push @{ $self->found_contacts }, $self->_current_service->{service_code};
        print "created contact for service code " . $self->_current_service->{service_code} . " for council @{[$self->_current_council->area_id]}\n";
    }
}

sub _add_meta_to_contact {
    my ( $self, $contact ) = @_;

    print "Fetching meta data for $self->_current_service->{service_code}\n";
    my $meta_data = $self->_current_open311->get_service_meta_info( $self->_current_service->{service_code} );

    if ( ref $meta_data->{ attributes }->{ attribute } eq 'HASH' ) {
        $meta_data->{ attributes }->{ attribute } = [
            $meta_data->{ attributes }->{ attribute }
        ];
    }

    # turn the data into something a bit more friendly to use
    my @meta =
        # remove trailing colon as we add this when we display so we don't want 2
        map { $_->{description} =~ s/:\s*//; $_ }
        # there is a display order and we only want to sort once
        sort { $a->{order} <=> $b->{order} }
        @{ $meta_data->{attributes}->{attribute} };

    # we add these later on from bromley so don't list them here
    # as we don't want to display them
    if ( $self->_current_council->area_id == 2482 ) {
        my %ignore = map { $_ => 1 } qw/
            service_request_id_ext
            requested_datetime
            report_url
            title
            last_name
            email
            easting
            northing
            report_title
            public_anonymity_required<commit_msg>add verbose option and apply to print statements<commit_after>package Open311::PopulateServiceList;

use Moose;
use LWP::Simple;
use XML::Simple;
use FixMyStreet::App;
use Open311;

has council_list => ( is => 'ro' );
has found_contacts => ( is => 'rw', default => sub { [] } );
has verbose => ( is => 'ro', default => 0 );

has _current_council => ( is => 'rw' );
has _current_open311 => ( is => 'rw' );
has _current_service => ( is => 'rw' );

my $council_list = FixMyStreet::App->model('DB::Open311conf');

sub process_councils {
    my $self = shift;

    while ( my $council = $self->council_list->next ) {
        next unless $council->endpoint;
        $self->_current_council( $council );
        $self->process_council;
    }
}

sub process_council {
    my $self = shift;

    my $open311 = Open311->new(
        endpoint => $self->_current_council->endpoint,
        jurisdiction => $self->_current_council->jurisdiction,
        api_key => $self->_current_council->api_key
    );

    $self->_current_open311( $open311 );
    $self->_check_endpoints;

    my $list = $open311->get_service_list;
    unless ( $list ) {
        warn "ERROR: no service list found for " . $self->_current_council->area_id . "\n";
        return;
    }
    $self->process_services( $list );
}



sub _check_endpoints {
    my $self = shift;

    # west berks end point not standard
    if ( $self->_current_council->area_id == 2619 ) {
        $self->_current_open311->endpoints(
            {
                services => 'Services',
                requests => 'Requests'
            }
        );
    }
}


sub process_services {
    my $self = shift;
    my $list = shift;

    $self->found_contacts( [] );
    foreach my $service ( @{ $list->{service} } ) {
        $self->_current_service( $service );
        $self->process_service;
    }
    $self->_delete_contacts_not_in_service_list;
}

sub process_service {
    my $self = shift;

    my $category = $self->_current_council->area_id == 2218 ? 
                    $self->_current_service->{description} : 
                    $self->_current_service->{service_name};

    print $self->_current_service->{service_code} . ': ' . $category .  "\n" if $self->verbose;
    my $contacts = FixMyStreet::App->model( 'DB::Contact')->search(
        {
            area_id => $self->_current_council->area_id,
            -OR => [
                email => $self->_current_service->{service_code},
                category => $category,
            ]
        }
    );

    if ( $contacts->count() > 1 ) {
        printf(
            "Multiple contacts for service code %s, category %s - Skipping\n",
            $self->_current_service->{service_code},
            $category,
        );

        # best to not mark them as deleted as we don't know what we're doing
        while ( my $contact = $contacts->next ) {
            push @{ $self->found_contacts }, $contact->email;
        }

        return;
    }

    my $contact = $contacts->first;

    if ( $contact ) {
        $self->_handle_existing_contact( $contact );
    } else {
        $self->_create_contact;
    }
}

sub _handle_existing_contact {
    my ( $self, $contact ) = @_;

    my $service_name = $self->_normalize_service_name;

    print $self->_current_council->area_id . " already has a contact for service code " . $self->_current_service->{service_code} . "\n" if $self->verbose;

    if ( $contact->deleted || $service_name ne $contact->category || $self->_current_service->{service_code} ne $contact->email ) {
        eval {
            $contact->update(
                {
                    category => $service_name,
                    email => $self->_current_service->{service_code},
                    confirmed => 1,
                    deleted => 0,
                    editor => $0,
                    whenedited => \'ms_current_timestamp()',
                    note => 'automatically undeleted by script',
                }
            );
        };

        if ( $@ ) {
            warn "Failed to update contact for service code " . $self->_current_service->{service_code} . " for council @{[$self->_current_council->area_id]}: $@\n";
            return;
        }
    }

    push @{ $self->found_contacts }, $self->_current_service->{service_code};
}

sub _create_contact {
    my $self = shift;

    my $service_name = $self->_normalize_service_name;

    my $contact;
    eval {
        $contact = FixMyStreet::App->model( 'DB::Contact')->create(
            {
                email => $self->_current_service->{service_code},
                area_id => $self->_current_council->area_id,
                category => $service_name,
                confirmed => 1,
                deleted => 0,
                editor => $0,
                whenedited => \'ms_current_timestamp()',
                note => 'created automatically by script',
            }
        );
    };

    if ( $@ ) {
        warn "Failed to create contact for service code " . $self->_current_service->{service_code} . " for council @{[$self->_current_council->area_id]}: $@\n";
        return;
    }

    if ( $contact and lc( $self->_current_service->{metadata} ) eq 'true' ) {
        $self->_add_meta_to_contact( $contact );
    }

    if ( $contact ) {
        push @{ $self->found_contacts }, $self->_current_service->{service_code};
        print "created contact for service code " . $self->_current_service->{service_code} . " for council @{[$self->_current_council->area_id]}\n" if $self->verbose;
    }
}

sub _add_meta_to_contact {
    my ( $self, $contact ) = @_;

    print "Fetching meta data for $self->_current_service->{service_code}\n" if $self->verbose;
    my $meta_data = $self->_current_open311->get_service_meta_info( $self->_current_service->{service_code} );

    if ( ref $meta_data->{ attributes }->{ attribute } eq 'HASH' ) {
        $meta_data->{ attributes }->{ attribute } = [
            $meta_data->{ attributes }->{ attribute }
        ];
    }

    # turn the data into something a bit more friendly to use
    my @meta =
        # remove trailing colon as we add this when we display so we don't want 2
        map { $_->{description} =~ s/:\s*//; $_ }
        # there is a display order and we only want to sort once
        sort { $a->{order} <=> $b->{order} }
        @{ $meta_data->{attributes}->{attribute} };

    # we add these later on from bromley so don't list them here
    # as we don't want to display them
    if ( $self->_current_council->area_id == 2482 ) {
        my %ignore = map { $_ => 1 } qw/
            service_request_id_ext
            requested_datetime
            report_url
            title
            last_name
            email
            easting
            northing
            report_title
            public_anonymity_required
{'fraction_non_alphanumeric': 0.1408199643493761, 'fraction_numerical': 0.008095662507427213, 'mean_word_length': 2.0890112411103465, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 308, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5700102', 'n_tokens_mistral': 4247, 'n_tokens_neox': 3967, 'n_words': 904}