<commit_before>package resources::metagenome;

use strict;
use warnings;
no warnings('once');

use URI::Escape;
use List::Util qw(first max min sum);
use POSIX qw(strftime floor);

use MGRAST::Metadata;
use MGRAST::Analysis;
use Conf;
use parent qw(resources::resource);

# Override parent constructor
sub new {
    my ($class, @args) = @_;

    # Call the constructor of the parent class
    my $self = $class->SUPER::new(@args);
    
    # Add name / attributes
    my %rights = $self->user ? map {$_, 1} grep {$_ ne '*'} @{$self->user->has_right_to(undef, 'view', 'metagenome')} : ();
    $self->{name} = "metagenome";
    $self->{mgdb} = undef;
    $self->{rights} = \%rights;
    $self->{cv} = { verbosity => {'minimal' => 1, 'mixs' => 1, 'metadata' => 1, 'stats' => 1, 'full' => 1},
                    direction => {'asc' => 1, 'desc' => 1},
                    status => {'both' => 1, 'public' => 1, 'private' => 1},
                    match => {'any' => 1, 'all' => 1}
    };
    $self->{instance} = { "id"       => [ 'string', 'unique metagenome identifier' ],
                          "url"      => [ 'uri', 'resource location of this object instance' ],
                          "name"     => [ 'string', 'name of metagenome' ],
                          "library"  => [ 'reference library', 'reference to the related library object' ],
                          "sample"   => [ 'reference sample', 'reference to the related sample object' ],
                          "project"  => [ 'reference project', 'reference to the project object' ],
                          "metadata" => [ 'hash', 'key value pairs describing all metadata' ],
                          "mixs"     => [ 'hash', 'key value pairs describing MIxS metadata' ],
                          "created"  => [ 'date', 'time the metagenome was first created' ],
                          "version"  => [ 'integer', 'version of the metagenome' ],
                          "status"   => [ 'cv', [['public', 'metagenome is public'],
						                         ['private', 'metagenome is private'] ] ],
						  "statistics" => [ 'hash', 'key value pairs describing statistics' ],
                          "sequence_type" => [ 'string', 'sequencing type' ],
                          "pipeline_parameters" => [ 'hash', 'key value pairs describing pipeline parameters' ]
    };
    $self->{query} = { "id"        => [ 'string', 'unique metagenome identifier' ],
                       "url"       => [ 'uri', 'resource location of this object instance' ],
                       "name"      => [ 'string', 'name of metagenome' ],
                       "biome"     => [ 'string', 'environmental biome, EnvO term' ],
                       "feature"   => [ 'string', 'environmental feature, EnvO term' ],
                       "material"  => [ 'string', 'environmental material, EnvO term' ],
                       "country"   => [ 'string', 'country where sample taken' ],
                       "location"  => [ 'string', 'location where sample taken' ],
                       "longitude" => [ 'string', 'longitude where sample taken' ],
                       "latitude"  => [ 'string', 'latitude where sample taken' ],
                       "created"   => [ 'date', 'time the metagenome was first created' ],
                       "status"    => [ 'cv', [['public', 'metagenome is public'],
						                       ['private', 'metagenome is private']] ],
					   "env_package_type" => [ 'string', 'environmental package of sample, GSC term' ],
					   "project_id"       => [ 'string', 'id of project containing metagenome' ],
                       "project_name"     => [ 'string', 'name of project containing metagenome' ],
                       "PI_firstname"     => [ 'string', 'principal investigator\'s first name' ],
                       "PI_lastname"      => [ 'string', 'principal investigator\'s last name' ],
                       "sequence_type"    => [ 'string', 'sequencing type' ],
                       "seq_method"       => [ 'string', 'sequencing method' ],
                       "collection_date"  => [ 'string', 'date sample collected' ]
    };
    return $self;
}

# resource is called without any parameters
# this method must return a description of the resource
sub info {
    my ($self) = @_;
    my $content = { 'name'          => $self->name,
                    'url'           => $self->cgi->url."/".$self->name,
                    'description'   => "A metagenome is an analyzed set sequences from a sample of some environment",
                    'type'          => 'object',
                    'documentation' => $self->cgi->url.'/api.html#'.$self->name,
                    'requests'      => [{ 'name'        => "info",
                                          'request'     => $self->cgi->url."/".$self->name,
                                          'description' => "Returns description of parameters and attributes.",
                                          'method'      => "GET",
                                          'type'        => "synchronous",
                                          'attributes'  => "self",
                                          'parameters'  => { 'options'  => {},
                                                             'required' => {},
                                                             'body'     => {} }
                                        },
                                        { 'name'        => "query",
                                          'request'     => $self->cgi->url."/".$self->name,
                                          'description' => "Returns a set of data matching the query criteria.",
                                          'example'     => [ $self->cgi->url."/".$self->name."?limit=20&order=name",
                          				                     'retrieve the first 20 metagenomes ordered by name' ],
                                          'method'      => "GET",
                                          'type'        => "synchronous",
                                          'attributes'  => { "next"    => ["uri","link to the previous set or null if this is the first set"],
                                                             "prev"    => ["uri","link to the next set or null if this is the last set"],
                                                             "order"   => ["string","name of the attribute the returned data is ordered by"],
                                                             "data"    => ["list", ["object", [$self->{query}, "metagenome object"] ]],
                                                             "limit"   => ["integer","maximum number of data items returned, default is 10"],
                                                             "offset"  => ["integer","zero based index of the first returned data item"],
                                                             "version" => ['integer', 'version of the object'],
                                                             "url"     => ['uri', 'resource location of this object instance'],
                                                             "total_count" => ["integer","total number of available data items"] },
                                          'parameters' => { 'options' => {
                                                                'metadata'  => ["string", "search parameter: query string for any metadata field"],
                                                                'md5'       => ["string", "search parameter: md5 checksum of feature sequence"],
                                                                'function'  => ["string", "search parameter: query string for function"],
                                                                'organism'  => ["string", "search parameter: query string for organism"],
                                                                'limit'     => ["integer", "maximum number of items requested"],
                                                                'offset'    => ["integer", "zero based index of the first data object to be returned"],
                                                                'order'     => ["string", "metagenome object field to sort by (default is id)"],
                                                                'direction' => ['cv', [['asc','sort by ascending order'],
                                                                                       ['desc','sort by descending order']]],
                                                                'match' => ['cv', [['all','return metagenomes that match all search parameters'],
                                                                                   ['any','return metagenomes that match any search parameters']]],
                                                                'status' => ['cv', [['both','returns all data (public and private) user has access to view'],
                                                                                    ['public','returns all public data'],
                                                                                    ['private','returns private data user has access to view']]],
                                                                'verbosity' => ['cv', [['minimal','returns only minimal information'],
                                                                                       ['mixs','returns all GSC MIxS metadata'],
                                                                                       ['metadata','returns minimal with metadata'],
                                                                                       ['stats','returns minimal with statistics'],
                                                                                       ['full','returns all metadata and statistics']] ] },
                                                            'required' => {},
                                                            'body'     => {} }
                                        },
                                        { 'name'        => "instance",
                                          'request'     => $self->cgi->url."/".$self->name."/{ID}",
                                          'description' => "Returns a single data object.",
                                          'example'     => [ $self->cgi->url."/".$self->name."/mgm4447943.3?verbosity=metadata",
                          				                     'retrieve all metadata for metagenome mgm4447943.3' ],
                                          'method'      => "GET",
                                          'type'        => "synchronous",
                                          'attributes'  => $self->{instance},
                                          'parameters'  => { 'options' => {
                                                                 'verbosity' => ['cv', [['minimal','returns only minimal information'],
                                                                                        ['metadata','returns minimal with metadata'],
                                                                                        ['stats','returns minimal with statistics'],
                                                                                        ['full','returns all metadata and statistics']]]
                                                                          },
                                                             'required' => { "id" => ["string","unique object identifier"] },
                                                             'body'     => {} }
                                        }] };
    $self->return_data($content);
}

# the resource is called with an id parameter
sub instance {
    my ($self) = @_;
    
    # check verbosity
    my $verb = $self->cgi->param('verbosity') || 'minimal';
    unless (exists $self->{cv}{verbosity}{$verb}) {
        $self->return_data({"ERROR" => "Invalid verbosity entered ($verb)."}, 404);
    }
    
    # get database
    my $master = $self->connect_to_datasource();
    my $rest = $self->rest;
    
    # overload id to be md5 of metagenome sequence file
    if (($rest->[0] eq 'md5') && (scalar(@$rest) > 1)) {
        my $data = [];
        my $jobs = $master->Job->get_objects( {file_checksum_raw => $rest->[1]} );
        if ($jobs && @$jobs) {
            my $valid_jobs = [];
            foreach my $job (@$jobs) {
                if ($job->{public} || exists($self->rights->{$job->{metagenome_id}}) || ($self->user && $self->user->has_star_right('view', 'metagenome'))) {
                    push @$valid_jobs, $job;
                }
            }
            # return cached if exists
            $self->return_cached();
            # prepare data
            $data = $self->prepare_data($valid_jobs, $verb);
        }
        my $obj = {
            version => 1,
            data => $data,
            total_count => scalar(@$data),
            md5 => $rest->[1],
            user => $self->user ? $self->user->login : 'public'
        };
        $self->return_data($obj, undef, 1); # cache this!
    }
    
    # check id format
    my (undef, $id) = $rest->[0] =~ /^(mgm)?(\d+\.\d+)$/;
    if ((! $id) && scalar(@$rest)) {
        $self->return_data( {"ERROR" => "invalid id format: " . $rest->[0]}, 400 );<commit_msg>fix metagenome lookup by md5<commit_after>package resources::metagenome;

use strict;
use warnings;
no warnings('once');

use URI::Escape;
use List::Util qw(first max min sum);
use POSIX qw(strftime floor);

use MGRAST::Metadata;
use MGRAST::Analysis;
use Conf;
use parent qw(resources::resource);

# Override parent constructor
sub new {
    my ($class, @args) = @_;

    # Call the constructor of the parent class
    my $self = $class->SUPER::new(@args);
    
    # Add name / attributes
    my %rights = $self->user ? map {$_, 1} grep {$_ ne '*'} @{$self->user->has_right_to(undef, 'view', 'metagenome')} : ();
    $self->{name} = "metagenome";
    $self->{mgdb} = undef;
    $self->{rights} = \%rights;
    $self->{cv} = { verbosity => {'minimal' => 1, 'mixs' => 1, 'metadata' => 1, 'stats' => 1, 'full' => 1},
                    direction => {'asc' => 1, 'desc' => 1},
                    status => {'both' => 1, 'public' => 1, 'private' => 1},
                    match => {'any' => 1, 'all' => 1}
    };
    $self->{instance} = { "id"       => [ 'string', 'unique metagenome identifier' ],
                          "url"      => [ 'uri', 'resource location of this object instance' ],
                          "name"     => [ 'string', 'name of metagenome' ],
                          "library"  => [ 'reference library', 'reference to the related library object' ],
                          "sample"   => [ 'reference sample', 'reference to the related sample object' ],
                          "project"  => [ 'reference project', 'reference to the project object' ],
                          "metadata" => [ 'hash', 'key value pairs describing all metadata' ],
                          "mixs"     => [ 'hash', 'key value pairs describing MIxS metadata' ],
                          "created"  => [ 'date', 'time the metagenome was first created' ],
                          "version"  => [ 'integer', 'version of the metagenome' ],
                          "status"   => [ 'cv', [['public', 'metagenome is public'],
						                         ['private', 'metagenome is private'] ] ],
						  "statistics" => [ 'hash', 'key value pairs describing statistics' ],
                          "sequence_type" => [ 'string', 'sequencing type' ],
                          "pipeline_parameters" => [ 'hash', 'key value pairs describing pipeline parameters' ]
    };
    $self->{query} = { "id"        => [ 'string', 'unique metagenome identifier' ],
                       "url"       => [ 'uri', 'resource location of this object instance' ],
                       "name"      => [ 'string', 'name of metagenome' ],
                       "biome"     => [ 'string', 'environmental biome, EnvO term' ],
                       "feature"   => [ 'string', 'environmental feature, EnvO term' ],
                       "material"  => [ 'string', 'environmental material, EnvO term' ],
                       "country"   => [ 'string', 'country where sample taken' ],
                       "location"  => [ 'string', 'location where sample taken' ],
                       "longitude" => [ 'string', 'longitude where sample taken' ],
                       "latitude"  => [ 'string', 'latitude where sample taken' ],
                       "created"   => [ 'date', 'time the metagenome was first created' ],
                       "status"    => [ 'cv', [['public', 'metagenome is public'],
						                       ['private', 'metagenome is private']] ],
					   "env_package_type" => [ 'string', 'environmental package of sample, GSC term' ],
					   "project_id"       => [ 'string', 'id of project containing metagenome' ],
                       "project_name"     => [ 'string', 'name of project containing metagenome' ],
                       "PI_firstname"     => [ 'string', 'principal investigator\'s first name' ],
                       "PI_lastname"      => [ 'string', 'principal investigator\'s last name' ],
                       "sequence_type"    => [ 'string', 'sequencing type' ],
                       "seq_method"       => [ 'string', 'sequencing method' ],
                       "collection_date"  => [ 'string', 'date sample collected' ]
    };
    return $self;
}

# resource is called without any parameters
# this method must return a description of the resource
sub info {
    my ($self) = @_;
    my $content = { 'name'          => $self->name,
                    'url'           => $self->cgi->url."/".$self->name,
                    'description'   => "A metagenome is an analyzed set sequences from a sample of some environment",
                    'type'          => 'object',
                    'documentation' => $self->cgi->url.'/api.html#'.$self->name,
                    'requests'      => [{ 'name'        => "info",
                                          'request'     => $self->cgi->url."/".$self->name,
                                          'description' => "Returns description of parameters and attributes.",
                                          'method'      => "GET",
                                          'type'        => "synchronous",
                                          'attributes'  => "self",
                                          'parameters'  => { 'options'  => {},
                                                             'required' => {},
                                                             'body'     => {} }
                                        },
                                        { 'name'        => "query",
                                          'request'     => $self->cgi->url."/".$self->name,
                                          'description' => "Returns a set of data matching the query criteria.",
                                          'example'     => [ $self->cgi->url."/".$self->name."?limit=20&order=name",
                          				                     'retrieve the first 20 metagenomes ordered by name' ],
                                          'method'      => "GET",
                                          'type'        => "synchronous",
                                          'attributes'  => { "next"    => ["uri","link to the previous set or null if this is the first set"],
                                                             "prev"    => ["uri","link to the next set or null if this is the last set"],
                                                             "order"   => ["string","name of the attribute the returned data is ordered by"],
                                                             "data"    => ["list", ["object", [$self->{query}, "metagenome object"] ]],
                                                             "limit"   => ["integer","maximum number of data items returned, default is 10"],
                                                             "offset"  => ["integer","zero based index of the first returned data item"],
                                                             "version" => ['integer', 'version of the object'],
                                                             "url"     => ['uri', 'resource location of this object instance'],
                                                             "total_count" => ["integer","total number of available data items"] },
                                          'parameters' => { 'options' => {
                                                                'metadata'  => ["string", "search parameter: query string for any metadata field"],
                                                                'md5'       => ["string", "search parameter: md5 checksum of feature sequence"],
                                                                'function'  => ["string", "search parameter: query string for function"],
                                                                'organism'  => ["string", "search parameter: query string for organism"],
                                                                'limit'     => ["integer", "maximum number of items requested"],
                                                                'offset'    => ["integer", "zero based index of the first data object to be returned"],
                                                                'order'     => ["string", "metagenome object field to sort by (default is id)"],
                                                                'direction' => ['cv', [['asc','sort by ascending order'],
                                                                                       ['desc','sort by descending order']]],
                                                                'match' => ['cv', [['all','return metagenomes that match all search parameters'],
                                                                                   ['any','return metagenomes that match any search parameters']]],
                                                                'status' => ['cv', [['both','returns all data (public and private) user has access to view'],
                                                                                    ['public','returns all public data'],
                                                                                    ['private','returns private data user has access to view']]],
                                                                'verbosity' => ['cv', [['minimal','returns only minimal information'],
                                                                                       ['mixs','returns all GSC MIxS metadata'],
                                                                                       ['metadata','returns minimal with metadata'],
                                                                                       ['stats','returns minimal with statistics'],
                                                                                       ['full','returns all metadata and statistics']] ] },
                                                            'required' => {},
                                                            'body'     => {} }
                                        },
                                        { 'name'        => "instance",
                                          'request'     => $self->cgi->url."/".$self->name."/{ID}",
                                          'description' => "Returns a single data object.",
                                          'example'     => [ $self->cgi->url."/".$self->name."/mgm4447943.3?verbosity=metadata",
                          				                     'retrieve all metadata for metagenome mgm4447943.3' ],
                                          'method'      => "GET",
                                          'type'        => "synchronous",
                                          'attributes'  => $self->{instance},
                                          'parameters'  => { 'options' => {
                                                                 'verbosity' => ['cv', [['minimal','returns only minimal information'],
                                                                                        ['metadata','returns minimal with metadata'],
                                                                                        ['stats','returns minimal with statistics'],
                                                                                        ['full','returns all metadata and statistics']]]
                                                                          },
                                                             'required' => { "id" => ["string","unique object identifier"] },
                                                             'body'     => {} }
                                        }] };
    $self->return_data($content);
}

# the resource is called with an id parameter
sub instance {
    my ($self) = @_;
    
    # check verbosity
    my $verb = $self->cgi->param('verbosity') || 'minimal';
    unless (exists $self->{cv}{verbosity}{$verb}) {
        $self->return_data({"ERROR" => "Invalid verbosity entered ($verb)."}, 404);
    }
    
    # get database
    my $master = $self->connect_to_datasource();
    my $rest = $self->rest;
    
    # overload id to be md5 of metagenome sequence file
    if (($rest->[0] eq 'md5') && (scalar(@$rest) > 1)) {
        my $data = [];
        my $jobs = $master->Job->get_objects( {file_checksum_raw => $rest->[1]} );
        if ($jobs && @$jobs) {
            my $valid_jobs = [];
            foreach my $job (@$jobs) {
                if ($job->{public} || exists($self->rights->{$job->{metagenome_id}}) || ($self->user && $self->user->has_star_right('view', 'metagenome'))) {
                    push @$valid_jobs, $job;
                }
            }
            # prepare data
            $data = $self->prepare_data($valid_jobs, $verb);
        }
        my $obj = {
            version => 1,
            data => $data,
            total_count => scalar(@$data),
            md5 => $rest->[1],
            user => $self->user ? $self->user->login : 'public'
        };
        $self->return_data($obj);
    }
    
    # check id format
    my (undef, $id) = $rest->[0] =~ /^(mgm)?(\d+\.\d+)$/;
    if ((! $id) && scalar(@$rest)) {
        $self->return_data( {"ERROR" => "invalid id format: " . $rest->[0]}, 400 );
{'fraction_non_alphanumeric': 0.13342462711425293, 'fraction_numerical': 0.0039971871645878825, 'mean_word_length': 1.0338727888596162, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 364, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 290, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5201994', 'n_tokens_mistral': 5782, 'n_tokens_neox': 5501, 'n_words': 1647}