<commit_before>package Geo::OGC::Service::Filter;

use Modern::Perl;
use Data::Dumper;
use Geo::OGC::Service;
use vars qw(@ISA);
push @ISA, qw(Geo::OGC::Service::Common);

require Exporter;
push @ISA, qw(Exporter);
our @EXPORT_OK = qw(filter2sql node2sql parse_tag parse_name strip get_integer latest_version);
our %EXPORT_TAGS = (all => \@EXPORT_OK);

our %gml_geometry_type = (
    Envelope => 1,
    Point => 1,
    MultiPoint => 1,
    LineString => 1,
    MultiLineString => 1,
    Polygon => 1,
    MultiPolygon => 1,
    MultiGeometry => 1,
    );

our %spatial2op = ( 
    Disjoint => 1,
    Equals => 1,
    Touches => 1,
    Within => 1,
    Overlaps => 1,
    Crosses => 1,
    Intersects => 1,
    Contains => 1
    ); # there are more in PostGIS

our %temporal_operators = (
    After => 1,
    Before => 1,
    Begins => 1,
    BegunBy => 1,
    TContains => 1,
    During => 1,
    TEquals => 1,
    TOverlaps => 1,
    Meets => 1,
    OverlappedBy => 1,
    MetBy => 1,
    EndedBy => 1
    );

my %functions = (
    abs => ['xs:int', [[int => 'xs:int']]],
    acos => ['xs:double', [[value => 'xs:double']]],
    Area => ['xs:double', [[geometry => 'gml:AbstractGeometryType']]],
    within => ['xs:boolean', [[geometry => 'gml:AbstractGeometryType'], [geometry => 'gml:AbstractGeometryType']]]
    ); # lots missing

sub Filter_Capabilities  {
    my ($self, $writer) = @_;
    my $ns = $self->{version} eq '2.0.0' ? 'fes' : 'ogc';
    $writer->open_element($ns.':Filter_Capabilities');

    # Conformance
    my %Constraints = ( 
        ImplementsQuery => [['ows:NoValues'], ['ows:DefaultValue' => 'TRUE']],
        ImplementsAdHocQuery => [['ows:NoValues'], ['ows:DefaultValue' => 'TRUE']],
        ImplementsFunctions => [['ows:NoValues'], ['ows:DefaultValue' => 'TRUE']],
        ImplementsMinStandardFilter => [['ows:NoValues'], ['ows:DefaultValue' => 'TRUE']],
        ImplementsStandardFilter => [['ows:NoValues'], ['ows:DefaultValue' => 'FALSE']],
        ImplementsMinSpatialFilter => [['ows:NoValues'], ['ows:DefaultValue' => 'TRUE']],
        ImplementsSpatialFilter => [['ows:NoValues'], ['ows:DefaultValue' => 'FALSE']],
        ImplementsMinTemporalFilter => [['ows:NoValues'], ['ows:DefaultValue' => 'TRUE']],
        ImplementsTemporalFilter => [['ows:NoValues'], ['ows:DefaultValue' => 'TRUE']],
        ImplementsVersionNav => [['ows:NoValues'], ['ows:DefaultValue' => 'FALSE']],
        ImplementsSorting => [['ows:AllowedValues' => [['ows:Value' => 'ASC'], ['ows:Value' => 'DESC']]], 
                              ['ows:DefaultValue' => 'ASC']],
        ImplementsExtendedOperators => [['ows:NoValues'], ['ows:DefaultValue' => 'FALSE']],
        );
    my @c;
    for my $key (keys %Constraints) {
        push @c, [$ns.':Constraint', {name=>$key}, $Constraints{$key}];
    }
    $writer->element($ns.':Conformance', \@c);

    my @ids;
    if ($ns eq 'ogc') {
        @ids = ([$ns.':FID']);
    } else {
        @ids = (['fes:ResourceIdentifier', {name => 'fes:ResourceId'}]);
    }
    $writer->element($ns.':Id_Capabilities', \@ids);

    my @operators = ();    
    for my $o (qw/LessThan GreaterThan LessThanOrEqualTo GreaterThanOrEqualTo EqualTo NotEqualTo Like Between Null/) {
        if ($ns eq 'ogc') {
            push @operators, [$ns.':ComparisonOperator', 'PropertyIs'.$o];
        } else {
            push @operators, [$ns.':ComparisonOperator', { name => 'PropertyIs'.$o}];
        }
    }
    $writer->element($ns.':Scalar_Capabilities', 
                [[$ns.':LogicalOperators'], # empty ?
                 [$ns.':ComparisonOperators', \@operators]]);

    my @operands = ();
    for my $o (keys %gml_geometry_type) {
        if ($ns eq 'ogc') {
            push @operands, [$ns.':GeometryOperand', 'gml:'.$o];
        } else {
            push @operands, [$ns.':GeometryOperand', { name => 'gml:'.$o }];
        }
    }
    @operators = ();
    my @op = keys %spatial2op;
    push @op, (qw/DWithin BBOX/);
    for my $o (@op) {
        push @operators, [$ns.':SpatialOperator', { name => $o }];
    }
    $writer->element($ns.':Spatial_Capabilities', 
                [[$ns.':GeometryOperands', \@operands],
                 [$ns.':SpatialOperators', \@operators]]);

    # Temporal_Capabilities

    @operands = ();
    for my $o (qw/TimeInstant TimePeriod/) {
        if ($ns eq 'ogc') {
            push @operands, [$ns.':GeometryOperand', 'gml:'.$o];
        } else {
            push @operands, [$ns.':GeometryOperand', { name => 'gml:'.$o }];
        }
    }
    @operators = ();
    @op = keys %temporal_operators;
    for my $o (@op) {
        push @operators, [$ns.':TemporalOperator', { name => $o }];
    }
    $writer->element($ns.':Temporal_Capabilities', 
                     [[$ns.':TemporalOperands', \@operands],
                      [$ns.':TemporalOperators', \@operators]]);

    # Functions
    
    my @functions;
    for my $f (sort keys %functions) {
        my @args = ();
        my $args = $functions{$f}[1];
        for my $arg (@$args) {
            push @args, [$ns.':Argument', { name => $arg->[0]}, [$ns.':Type' => $arg->[1]]];
        }
        push @functions, [$ns.':Function', { name => $f }, 
                          [[$ns.':Returns' => $functions{$f}[0]] ,[$ns.':Arguments' => \@args]]];
    }
    $writer->element($ns.':Functions', \@functions);
    
    $writer->close_element;
}

sub error {
    my ($self, $msg, $headers) = @_;
    if (!$msg->{debug}) {
        Geo::OGC::Service::error($self->{responder}, $msg, $headers);
    } else {
        my $json = JSON->new;
        $json->allow_blessed([1]);
        my $writer = $self->{responder}->([200, [ 'Content-Type' => 'application/json',
                                                  'Content-Encoding' => 'UTF-8' ]]);
        $writer->write($json->encode($msg->{debug}));
        $writer->close;
    }
}

sub log {
    my ($self, $msg) = @_;
    say STDERR Dumper($msg);
}

# convert OGC Filter XML to SQL
sub filter2sql {
    my ($node, $type) = @_;
    if (!$node) {
        return "";
    }
    my ($ns, $name) = parse_tag($node);

    if ($name eq 'Literal') {
        return "'".$node->firstChild->data."'";

    } elsif ($name eq 'PropertyName') {
        my $ref = strip($node->textContent);
        $ref = $type->{GeometryColumn} if $ref eq 'geometryProperty';
        return '"'.$ref.'"';

    } elsif ($name eq 'ValueReference') {
        my $ref = strip($node->textContent);
        $ref =~ s/^\w+://;
        $ref = $type->{GeometryColumn} if $ref eq 'geometryProperty';
        return '"'.$ref.'"';

    } elsif ($name eq 'ResourceId') {
        my $id = $node->getAttribute('rid');
        return $type->{"gml:id"}." = '$id'"; # $type->{"gml:id"} may not be set!

    } elsif ($name eq 'FeatureId') {
        my $id = $node->getAttribute('fid');
        return $type->{"gml:id"}." = '$id'"; # $type->{"gml:id"} may not be set!

    } elsif ($name eq 'GmlObjectId') {
        my $id = $node->getAttribute('gml:id');
        return $type->{"gml:id"}." = '$id'"; # $type->{"gml:id"} may not be set!

    } elsif ($name eq 'PropertyIsEqualTo') {
        $node = $node->firstChild;
        return '('.filter2sql($node, $type).' = '.filter2sql($node->nextSibling, $type).')';

    } elsif ($name eq 'PropertyIsNotEqualTo') {
        $node = $node->firstChild;
        return '('.filter2sql($node, $type).' != '.filter2sql($node->nextSibling, $type).')';

    } elsif ($name eq 'PropertyIsLessThan') {
        $node = $node->firstChild;
        return '('.filter2sql($node, $type).' < '.filter2sql($node->nextSibling, $type).')';

    } elsif ($name eq 'PropertyIsGreaterThan') {
        $node = $node->firstChild;
        return '('.filter2sql($node, $type).' > '.filter2sql($node->nextSibling, $type).')';

    } elsif ($name eq 'PropertyIsLessThanOrEqualTo') {
        $node = $node->firstChild;
        return '('.filter2sql($node, $type).' <= '.filter2sql($node->nextSibling, $type).')';

    } elsif ($name eq 'PropertyIsGreaterThanOrEqualTo') {
        $node = $node->firstChild;
        return '('.filter2sql($node, $type).' >= '.filter2sql($node->nextSibling, $type).')';

    } elsif ($name eq 'PropertyIsBetween') {
        $node = $node->firstChild;
        my $property = filter2sql($node->firstChild, $type);
        $node = $node->nextSibling;
        return '('.$property.' >= '.filter2sql($node, $type).' AND '.$property.'<='.filter2sql($node->nextSibling).')';

    } elsif ($name eq 'PropertyIsLike') {
        $node = $node->firstChild;
        return '('.filter2sql($node, $type).' ~ '.filter2sql($node->nextSibling, $type).')';

    } elsif ($name eq 'PropertyIsNull') {
        return '('.filter2sql($node->firstChild, $type).' ISNULL)';

    } elsif ($name eq 'And') {
        $node = $node->firstChild;
        my $p = '('.filter2sql($node, $type);
        while ($node = $node->nextSibling) {
            $p .= ' AND '.filter2sql($node, $type);
        }
        return $p.')';

    } elsif ($name eq 'Or') {
        $node = $node->firstChild;
        my $p = '('.filter2sql($node, $type);
        while ($node = $node->nextSibling) {
            $p .= ' OR '.filter2sql($node, $type);
        }
        return $p.')';
    } elsif ($name eq 'Not') {
        return '(NOT '.filter2sql($node->firstChild, $type).')';

    } elsif ($ns eq 'gml' and $gml_geometry_type{$name}) {
        return node2sql($node);

    } elsif ($name eq 'BBOX') {
        $node = $node->firstChild;
        my ($ns, $n) = parse_tag($node);
        my $property;
        if ($n eq 'Envelope') {
            $property = 'GeometryColumn';
        } else {
            $property = filter2sql($node, $type);
            $node = $node->nextSibling;
        }
        my $envelope = filter2sql($node, $type);
        if (exists $type->{SRID} && $type->{SRID} >= 0) {
            $envelope = "ST_Transform($envelope,$type->{SRID})";
        }
        return "($property && $envelope)";
        
    } elsif ($spatial2op{$name}) {
        $node = $node->firstChild;
        my $geom1 = filter2sql($node, $type);
        $node = $node->nextSibling;
        my $geom2 = filter2sql($node, $type);
        return "ST_$name($geom1, $geom2)";

    } elsif ($name eq 'DWithin') {
        $node = $node->firstChild;
        my $geom1 = filter2sql($node, $type);
        $node = $node->nextSibling;
        my $geom2 = filter2sql($node, $type);
        $node = $node->nextSibling;
        my $dist = filter2sql($node, $type);<commit_msg>SRID <= 0 is unknown (in PostGIS)...<commit_after>package Geo::OGC::Service::Filter;

use Modern::Perl;
use Data::Dumper;
use Geo::OGC::Service;
use vars qw(@ISA);
push @ISA, qw(Geo::OGC::Service::Common);

require Exporter;
push @ISA, qw(Exporter);
our @EXPORT_OK = qw(filter2sql node2sql parse_tag parse_name strip get_integer latest_version);
our %EXPORT_TAGS = (all => \@EXPORT_OK);

our %gml_geometry_type = (
    Envelope => 1,
    Point => 1,
    MultiPoint => 1,
    LineString => 1,
    MultiLineString => 1,
    Polygon => 1,
    MultiPolygon => 1,
    MultiGeometry => 1,
    );

our %spatial2op = ( 
    Disjoint => 1,
    Equals => 1,
    Touches => 1,
    Within => 1,
    Overlaps => 1,
    Crosses => 1,
    Intersects => 1,
    Contains => 1
    ); # there are more in PostGIS

our %temporal_operators = (
    After => 1,
    Before => 1,
    Begins => 1,
    BegunBy => 1,
    TContains => 1,
    During => 1,
    TEquals => 1,
    TOverlaps => 1,
    Meets => 1,
    OverlappedBy => 1,
    MetBy => 1,
    EndedBy => 1
    );

my %functions = (
    abs => ['xs:int', [[int => 'xs:int']]],
    acos => ['xs:double', [[value => 'xs:double']]],
    Area => ['xs:double', [[geometry => 'gml:AbstractGeometryType']]],
    within => ['xs:boolean', [[geometry => 'gml:AbstractGeometryType'], [geometry => 'gml:AbstractGeometryType']]]
    ); # lots missing

sub Filter_Capabilities  {
    my ($self, $writer) = @_;
    my $ns = $self->{version} eq '2.0.0' ? 'fes' : 'ogc';
    $writer->open_element($ns.':Filter_Capabilities');

    # Conformance
    my %Constraints = ( 
        ImplementsQuery => [['ows:NoValues'], ['ows:DefaultValue' => 'TRUE']],
        ImplementsAdHocQuery => [['ows:NoValues'], ['ows:DefaultValue' => 'TRUE']],
        ImplementsFunctions => [['ows:NoValues'], ['ows:DefaultValue' => 'TRUE']],
        ImplementsMinStandardFilter => [['ows:NoValues'], ['ows:DefaultValue' => 'TRUE']],
        ImplementsStandardFilter => [['ows:NoValues'], ['ows:DefaultValue' => 'FALSE']],
        ImplementsMinSpatialFilter => [['ows:NoValues'], ['ows:DefaultValue' => 'TRUE']],
        ImplementsSpatialFilter => [['ows:NoValues'], ['ows:DefaultValue' => 'FALSE']],
        ImplementsMinTemporalFilter => [['ows:NoValues'], ['ows:DefaultValue' => 'TRUE']],
        ImplementsTemporalFilter => [['ows:NoValues'], ['ows:DefaultValue' => 'TRUE']],
        ImplementsVersionNav => [['ows:NoValues'], ['ows:DefaultValue' => 'FALSE']],
        ImplementsSorting => [['ows:AllowedValues' => [['ows:Value' => 'ASC'], ['ows:Value' => 'DESC']]], 
                              ['ows:DefaultValue' => 'ASC']],
        ImplementsExtendedOperators => [['ows:NoValues'], ['ows:DefaultValue' => 'FALSE']],
        );
    my @c;
    for my $key (keys %Constraints) {
        push @c, [$ns.':Constraint', {name=>$key}, $Constraints{$key}];
    }
    $writer->element($ns.':Conformance', \@c);

    my @ids;
    if ($ns eq 'ogc') {
        @ids = ([$ns.':FID']);
    } else {
        @ids = (['fes:ResourceIdentifier', {name => 'fes:ResourceId'}]);
    }
    $writer->element($ns.':Id_Capabilities', \@ids);

    my @operators = ();    
    for my $o (qw/LessThan GreaterThan LessThanOrEqualTo GreaterThanOrEqualTo EqualTo NotEqualTo Like Between Null/) {
        if ($ns eq 'ogc') {
            push @operators, [$ns.':ComparisonOperator', 'PropertyIs'.$o];
        } else {
            push @operators, [$ns.':ComparisonOperator', { name => 'PropertyIs'.$o}];
        }
    }
    $writer->element($ns.':Scalar_Capabilities', 
                [[$ns.':LogicalOperators'], # empty ?
                 [$ns.':ComparisonOperators', \@operators]]);

    my @operands = ();
    for my $o (keys %gml_geometry_type) {
        if ($ns eq 'ogc') {
            push @operands, [$ns.':GeometryOperand', 'gml:'.$o];
        } else {
            push @operands, [$ns.':GeometryOperand', { name => 'gml:'.$o }];
        }
    }
    @operators = ();
    my @op = keys %spatial2op;
    push @op, (qw/DWithin BBOX/);
    for my $o (@op) {
        push @operators, [$ns.':SpatialOperator', { name => $o }];
    }
    $writer->element($ns.':Spatial_Capabilities', 
                [[$ns.':GeometryOperands', \@operands],
                 [$ns.':SpatialOperators', \@operators]]);

    # Temporal_Capabilities

    @operands = ();
    for my $o (qw/TimeInstant TimePeriod/) {
        if ($ns eq 'ogc') {
            push @operands, [$ns.':GeometryOperand', 'gml:'.$o];
        } else {
            push @operands, [$ns.':GeometryOperand', { name => 'gml:'.$o }];
        }
    }
    @operators = ();
    @op = keys %temporal_operators;
    for my $o (@op) {
        push @operators, [$ns.':TemporalOperator', { name => $o }];
    }
    $writer->element($ns.':Temporal_Capabilities', 
                     [[$ns.':TemporalOperands', \@operands],
                      [$ns.':TemporalOperators', \@operators]]);

    # Functions
    
    my @functions;
    for my $f (sort keys %functions) {
        my @args = ();
        my $args = $functions{$f}[1];
        for my $arg (@$args) {
            push @args, [$ns.':Argument', { name => $arg->[0]}, [$ns.':Type' => $arg->[1]]];
        }
        push @functions, [$ns.':Function', { name => $f }, 
                          [[$ns.':Returns' => $functions{$f}[0]] ,[$ns.':Arguments' => \@args]]];
    }
    $writer->element($ns.':Functions', \@functions);
    
    $writer->close_element;
}

sub error {
    my ($self, $msg, $headers) = @_;
    if (!$msg->{debug}) {
        Geo::OGC::Service::error($self->{responder}, $msg, $headers);
    } else {
        my $json = JSON->new;
        $json->allow_blessed([1]);
        my $writer = $self->{responder}->([200, [ 'Content-Type' => 'application/json',
                                                  'Content-Encoding' => 'UTF-8' ]]);
        $writer->write($json->encode($msg->{debug}));
        $writer->close;
    }
}

sub log {
    my ($self, $msg) = @_;
    say STDERR Dumper($msg);
}

# convert OGC Filter XML to SQL
sub filter2sql {
    my ($node, $type) = @_;
    if (!$node) {
        return "";
    }
    my ($ns, $name) = parse_tag($node);

    if ($name eq 'Literal') {
        return "'".$node->firstChild->data."'";

    } elsif ($name eq 'PropertyName') {
        my $ref = strip($node->textContent);
        $ref = $type->{GeometryColumn} if $ref eq 'geometryProperty';
        return '"'.$ref.'"';

    } elsif ($name eq 'ValueReference') {
        my $ref = strip($node->textContent);
        $ref =~ s/^\w+://;
        $ref = $type->{GeometryColumn} if $ref eq 'geometryProperty';
        return '"'.$ref.'"';

    } elsif ($name eq 'ResourceId') {
        my $id = $node->getAttribute('rid');
        return $type->{"gml:id"}." = '$id'"; # $type->{"gml:id"} may not be set!

    } elsif ($name eq 'FeatureId') {
        my $id = $node->getAttribute('fid');
        return $type->{"gml:id"}." = '$id'"; # $type->{"gml:id"} may not be set!

    } elsif ($name eq 'GmlObjectId') {
        my $id = $node->getAttribute('gml:id');
        return $type->{"gml:id"}." = '$id'"; # $type->{"gml:id"} may not be set!

    } elsif ($name eq 'PropertyIsEqualTo') {
        $node = $node->firstChild;
        return '('.filter2sql($node, $type).' = '.filter2sql($node->nextSibling, $type).')';

    } elsif ($name eq 'PropertyIsNotEqualTo') {
        $node = $node->firstChild;
        return '('.filter2sql($node, $type).' != '.filter2sql($node->nextSibling, $type).')';

    } elsif ($name eq 'PropertyIsLessThan') {
        $node = $node->firstChild;
        return '('.filter2sql($node, $type).' < '.filter2sql($node->nextSibling, $type).')';

    } elsif ($name eq 'PropertyIsGreaterThan') {
        $node = $node->firstChild;
        return '('.filter2sql($node, $type).' > '.filter2sql($node->nextSibling, $type).')';

    } elsif ($name eq 'PropertyIsLessThanOrEqualTo') {
        $node = $node->firstChild;
        return '('.filter2sql($node, $type).' <= '.filter2sql($node->nextSibling, $type).')';

    } elsif ($name eq 'PropertyIsGreaterThanOrEqualTo') {
        $node = $node->firstChild;
        return '('.filter2sql($node, $type).' >= '.filter2sql($node->nextSibling, $type).')';

    } elsif ($name eq 'PropertyIsBetween') {
        $node = $node->firstChild;
        my $property = filter2sql($node->firstChild, $type);
        $node = $node->nextSibling;
        return '('.$property.' >= '.filter2sql($node, $type).' AND '.$property.'<='.filter2sql($node->nextSibling).')';

    } elsif ($name eq 'PropertyIsLike') {
        $node = $node->firstChild;
        return '('.filter2sql($node, $type).' ~ '.filter2sql($node->nextSibling, $type).')';

    } elsif ($name eq 'PropertyIsNull') {
        return '('.filter2sql($node->firstChild, $type).' ISNULL)';

    } elsif ($name eq 'And') {
        $node = $node->firstChild;
        my $p = '('.filter2sql($node, $type);
        while ($node = $node->nextSibling) {
            $p .= ' AND '.filter2sql($node, $type);
        }
        return $p.')';

    } elsif ($name eq 'Or') {
        $node = $node->firstChild;
        my $p = '('.filter2sql($node, $type);
        while ($node = $node->nextSibling) {
            $p .= ' OR '.filter2sql($node, $type);
        }
        return $p.')';
    } elsif ($name eq 'Not') {
        return '(NOT '.filter2sql($node->firstChild, $type).')';

    } elsif ($ns eq 'gml' and $gml_geometry_type{$name}) {
        return node2sql($node);

    } elsif ($name eq 'BBOX') {
        $node = $node->firstChild;
        my ($ns, $n) = parse_tag($node);
        my $property;
        if ($n eq 'Envelope') {
            $property = 'GeometryColumn';
        } else {
            $property = filter2sql($node, $type);
            $node = $node->nextSibling;
        }
        my $envelope = filter2sql($node, $type);
        if (exists $type->{SRID} && $type->{SRID} > 0) {
            $envelope = "ST_Transform($envelope,$type->{SRID})";
        }
        return "($property && $envelope)";
        
    } elsif ($spatial2op{$name}) {
        $node = $node->firstChild;
        my $geom1 = filter2sql($node, $type);
        $node = $node->nextSibling;
        my $geom2 = filter2sql($node, $type);
        return "ST_$name($geom1, $geom2)";

    } elsif ($name eq 'DWithin') {
        $node = $node->firstChild;
        my $geom1 = filter2sql($node, $type);
        $node = $node->nextSibling;
        my $geom2 = filter2sql($node, $type);
        $node = $node->nextSibling;
        my $dist = filter2sql($node, $type);
{'fraction_non_alphanumeric': 0.2101304389222127, 'fraction_numerical': 0.008045320384652005, 'mean_word_length': 2.5508789722785665, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 309, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1182807', 'n_tokens_mistral': 7236, 'n_tokens_neox': 7128, 'n_words': 1482}