<commit_before>#!/usr/bin/env perl

=head1 LICENSE

Copyright [1999-2014] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=cut

# Script to dump Ensembl triples. It was lashed together rapidly, and could stand to use a library for the writing of triples, with accommodation for the scale of the data involved.
# Requires installation of Ensembl Core and Compara APIs, as well as dependencies such as BioPerl


use strict;

use Getopt::Long;
use Pod::Usage;
use Bio::EnsEMBL::Registry;
use Bio::EnsEMBL::DBSQL::DBAdaptor;
use Digest::MD5 qw(md5_hex);

my $species = '';
my $version = '';
my $path;
my $virtgraph;
my $man;
my $limit;
my $help;
#"ensembldb.ensembl.org";
my $host = "mysql-ensembl-mirror.ebi.ac.uk"; 
my $port = "4240";
my $genome = "";

GetOptions (
    'species=s' => \$species,
    'version=s' => \$version,
    'virtgraph' => \$virtgraph,
    'out=s' => \$path,
    'limit=i' => \$limit,
    'host=s' => \$host,
    'port=i' => \$port,
    'genome=s' => \$genome,
    'help|?' => \$help, 
    man => \$man
    ) or pod2usage(2);
pod2usage(1) if $help;
pod2usage(-exitval => 0, -verbose => 2) if $man;

if (!$species || !$version) {
    pod2usage(1);
}
print STDOUT "Generating mappings using Ensembl core DB $species version $version\n";

my $outfile = ${species}.'_'.${version}.'_xrefs.ttl';
if ($path) {
    $path = $path."/";
    $outfile = $path.$outfile;
}

# create the output file
open OUT, ">$outfile" || die "Can't open out file $outfile\n";

my %prefix = (
  ensembl => 'http://rdf.ebi.ac.uk/resource/ensembl/',
  transcript => 'http://rdf.ebi.ac.uk/resource/ensembl.transcript/',
  protein => 'http://rdf.ebi.ac.uk/resource/ensembl.protein/',
  exon => 'http://rdf.ebi.ac.uk/resource/ensembl.exon/',
  dataset => 'http://rdf.ebi.ac.uk/dataset/ensembl/',
  term => 'http://rdf.ebi.ac.uk/terms/ensembl/',
  rdfs => 'http://www.w3.org/2000/01/rdf-schema#',
  void => 'http://rdfs.org/ns/void#',
  sio => 'http://semanticscience.org/resource/',
  dc => 'http://purl.org/dc/terms/',
  rdf => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
  prov => 'http://www.w3.org/ns/prov#',
  faldo => 'http://biohackathon.org/resource/faldo#',
  obo => 'http://purl.obolibrary.org/obo/',
  skos => 'http://www.w3.org/2004/02/skos/core#',
  taxon => 'http://identifiers.org/taxonomy/',
);

foreach (keys %prefix) {
    triple('@prefix',$_.':',u($prefix{$_}) );
}

# Choice of database host is a factor in how fast the script runs. Try to find your nearest mirror, and check the database version before running.
Bio::EnsEMBL::Registry->load_registry_from_db(
  -host => $host,
#  -host => 'ensembldb.ensembl.org',
  -user => 'anonymous',
  -port => $port,
  -db_version => $version,
#  -no_cache => 1,
);

my $ga = Bio::EnsEMBL::Registry->get_adaptor($species,'Core','Gene');

my $meta = Bio::EnsEMBL::Registry->get_adaptor($species,'Core','MetaContainer');

my $pfa = Bio::EnsEMBL::Registry->get_adaptor($species,"funcgen","ProbeFeature");
my $aa = Bio::EnsEMBL::Registry->get_adaptor($species,"funcgen","Array");
my $pba = Bio::EnsEMBL::Registry->get_adaptor($species,"funcgen","ProbeSet");

# create a map of taxon id to species name, we will create some triples for these at the end

# get the taxon id for this species 
my $taxon_id = $meta->get_taxonomy_id;
my $scientific_name = $meta->get_scientific_name;

my %dbname2type;
my %dbname2base;
my %dbname2short;
my %dbid2regex;
my %ignore;

# check if we want to create a virtuoso graph file for this output file
my $versionGraphUri = "http://rdf.ebi.ac.uk/dataset/ensembl/".$version;
my $graphUri = $versionGraphUri."/".$taxon_id;
if ($virtgraph) {
    my $graphFile = $outfile.'.graph';
    open GRAPH, ">$graphFile" || die "Can't create the virtuoso graph file\n";
    print GRAPH $graphUri;
    close GRAPH;
    # make the species graph a subgraph of the version graph
    triple (u($graphUri), '<http://www.w3.org/2004/03/trix/rdfg-1/subGraphOf>', u($versionGraphUri)); 
}


# read config
open DBNAME , '<xref_config.txt' || die "can't open mapping config file\n";
while (<DBNAME>) {
    next if $. < 2; # Skip first line

    chomp;
    my $line = $_;
    my @split = split /\t/, $line;
    my $dbname = $split[0];
    my $shortname = $split[2];
    my $baseUri = $split[3];
    my $typeUri = $split[4];
    my $na = $split[5];
    my $regex = $split[6];
    
    if ($na) {
	$ignore{$dbname} =1;
    }
    if ($shortname) {
	$dbname2short{$dbname} = $shortname;
    }
    if ($regex) {
	$dbid2regex{$dbname} = $regex;
    }
    if ($typeUri) {
	$dbname2type{$dbname} = $typeUri;
    }
    if ($baseUri) {
	$dbname2base{$dbname} = $baseUri;
    }
}
close DBNAME;

my %relations;
my %ensemblSources;
my $genes = $ga->fetch_all();
#my $gene = $ga->fetch_by_stable_id ('ENSG00000105393');
#print "GENE ", $gene->stable_id(), "\n";
my $count = 0;
while (my $gene = shift @$genes) {
    $count++;
    print_DBEntries( $gene->get_all_DBEntries() , $gene, undef, $gene->biotype(), 'ensembl');
    
    foreach my $transcript ( @{ $gene->get_all_Transcripts() } ) {
	print_DBEntries( $transcript->get_all_DBEntries() , $gene, 'INFERRED_FROM_TRANSCRIPT', $gene->biotype(), 'ensembl');
	print_DBEntries( $transcript->get_all_DBEntries() , $transcript,  undef, "transcript", "transcript");
#	print_Probe_Features ($transcript);
	
	if ( defined $transcript->translation() ) {
	    my $translation = $transcript->translation();
	    print_DBEntries( $translation->get_all_DBEntries() , $gene, 'INFERRED_FROM_TRANSLATION', $gene->biotype());
	    print_DBEntries( $translation->get_all_DBEntries(), $translation, undef, "protein", "protein");
	}
    }
  last if ($limit && $count == $limit);
}

# print relation assertions as sub property of skos:related
#open RELOUT, ">${path}${species}_xref_relations.txt" || die "can't open ${species} xref relations\n";
open SRCOUT, ">${path}${species}_xref_sources.txt" || die "can't open ${species} xref relations\n";<commit_msg>updated void, fixed bug in mapping script<commit_after>#!/usr/bin/env perl

=head1 LICENSE

Copyright [1999-2014] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=cut

# Script to dump Ensembl triples. It was lashed together rapidly, and could stand to use a library for the writing of triples, with accommodation for the scale of the data involved.
# Requires installation of Ensembl Core and Compara APIs, as well as dependencies such as BioPerl


use strict;

use Getopt::Long;
use Pod::Usage;
use Bio::EnsEMBL::Registry;
use Bio::EnsEMBL::DBSQL::DBAdaptor;
use Digest::MD5 qw(md5_hex);

my $species = '';
my $version = '';
my $path;
my $virtgraph;
my $man;
my $limit;
my $help;
#"ensembldb.ensembl.org";
my $host = "mysql-ensembl-mirror.ebi.ac.uk"; 
my $port = "4240";
my $genome = "";

GetOptions (
    'species=s' => \$species,
    'version=s' => \$version,
    'virtgraph' => \$virtgraph,
    'out=s' => \$path,
    'limit=i' => \$limit,
    'host=s' => \$host,
    'port=i' => \$port,
    'genome=s' => \$genome,
    'help|?' => \$help, 
    man => \$man
    ) or pod2usage(2);
pod2usage(1) if $help;
pod2usage(-exitval => 0, -verbose => 2) if $man;

if (!$species || !$version) {
    pod2usage(1);
}
print STDOUT "Generating mappings using Ensembl core DB $species version $version\n";

my $outfile = ${species}.'_'.${version}.'_xrefs.ttl';
if ($path) {
    $path = $path."/";
    $outfile = $path.$outfile;
}

# create the output file
open OUT, ">$outfile" || die "Can't open out file $outfile\n";

my %prefix = (
  ensembl => 'http://rdf.ebi.ac.uk/resource/ensembl/',
  transcript => 'http://rdf.ebi.ac.uk/resource/ensembl.transcript/',
  protein => 'http://rdf.ebi.ac.uk/resource/ensembl.protein/',
  exon => 'http://rdf.ebi.ac.uk/resource/ensembl.exon/',
  dataset => 'http://rdf.ebi.ac.uk/dataset/ensembl/',
  term => 'http://rdf.ebi.ac.uk/terms/ensembl/',
  rdfs => 'http://www.w3.org/2000/01/rdf-schema#',
  void => 'http://rdfs.org/ns/void#',
  sio => 'http://semanticscience.org/resource/',
  dc => 'http://purl.org/dc/terms/',
  rdf => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
  prov => 'http://www.w3.org/ns/prov#',
  faldo => 'http://biohackathon.org/resource/faldo#',
  obo => 'http://purl.obolibrary.org/obo/',
  skos => 'http://www.w3.org/2004/02/skos/core#',
  taxon => 'http://identifiers.org/taxonomy/',
);

foreach (keys %prefix) {
    triple('@prefix',$_.':',u($prefix{$_}) );
}

# Choice of database host is a factor in how fast the script runs. Try to find your nearest mirror, and check the database version before running.
Bio::EnsEMBL::Registry->load_registry_from_db(
  -host => $host,
#  -host => 'ensembldb.ensembl.org',
  -user => 'anonymous',
  -port => $port,
  -db_version => $version,
#  -no_cache => 1,
);

my $ga = Bio::EnsEMBL::Registry->get_adaptor($species,'Core','Gene');

my $meta = Bio::EnsEMBL::Registry->get_adaptor($species,'Core','MetaContainer');

my $pfa = Bio::EnsEMBL::Registry->get_adaptor($species,"funcgen","ProbeFeature");
my $aa = Bio::EnsEMBL::Registry->get_adaptor($species,"funcgen","Array");
my $pba = Bio::EnsEMBL::Registry->get_adaptor($species,"funcgen","ProbeSet");

# create a map of taxon id to species name, we will create some triples for these at the end

# get the taxon id for this species 
my $taxon_id = $meta->get_taxonomy_id;
my $scientific_name = $meta->get_scientific_name;

my %dbname2type;
my %dbname2base;
my %dbname2short;
my %dbid2regex;
my %ignore;

# check if we want to create a virtuoso graph file for this output file
my $versionGraphUri = "http://rdf.ebi.ac.uk/dataset/ensembl/".$version;
my $graphUri = $versionGraphUri."/".$taxon_id;
if ($virtgraph) {
    my $graphFile = $outfile.'.graph';
    open GRAPH, ">$graphFile" || die "Can't create the virtuoso graph file\n";
    print GRAPH $graphUri;
    close GRAPH;
    # make the species graph a subgraph of the version graph
    triple (u($graphUri), '<http://www.w3.org/2004/03/trix/rdfg-1/subGraphOf>', u($versionGraphUri)); 
}


# read config
open DBNAME , '<xref_config.txt' || die "can't open mapping config file\n";
while (<DBNAME>) {
    next if $. < 2; # Skip first line

    chomp;
    my $line = $_;
    my @split = split /\t/, $line;
    my $dbname = $split[0];
    my $shortname = $split[2];
    my $baseUri = $split[3];
    my $typeUri = $split[4];
    my $na = $split[5];
    my $regex = $split[6];
    
    if ($na) {
	$ignore{$dbname} =1;
    }
    if ($shortname) {
	$dbname2short{$dbname} = $shortname;
    }
    if ($regex) {
	$dbid2regex{$dbname} = $regex;
    }
    if ($typeUri) {
	$dbname2type{$dbname} = $typeUri;
    }
    if ($baseUri) {
	$dbname2base{$dbname} = $baseUri;
    }
}
close DBNAME;

my %relations;
my %ensemblSources;
my $genes = $ga->fetch_all();
#my $gene = $ga->fetch_by_stable_id ('ENSG00000105393');
#print "GENE ", $gene->stable_id(), "\n";
my $count = 0;
while (my $gene = shift @$genes) {
    $count++;
    print_DBEntries( $gene->get_all_DBEntries() , $gene, undef, $gene->biotype(), 'ensembl');
    
    foreach my $transcript ( @{ $gene->get_all_Transcripts() } ) {
	print_DBEntries( $transcript->get_all_DBEntries() , $gene, 'INFERRED_FROM_TRANSCRIPT', $gene->biotype(), 'ensembl');
	print_DBEntries( $transcript->get_all_DBEntries() , $transcript,  undef, "transcript", "transcript");
#	print_Probe_Features ($transcript);
	
	if ( defined $transcript->translation() ) {
	    my $translation = $transcript->translation();
	    print_DBEntries( $translation->get_all_DBEntries() , $gene, 'INFERRED_FROM_TRANSLATION', $gene->biotype(), 'ensembl');
	    print_DBEntries( $translation->get_all_DBEntries(), $translation, undef, "protein", "protein");
	}
    }
  last if ($limit && $count == $limit);
}

# print relation assertions as sub property of skos:related
#open RELOUT, ">${path}${species}_xref_relations.txt" || die "can't open ${species} xref relations\n";
open SRCOUT, ">${path}${species}_xref_sources.txt" || die "can't open ${species} xref relations\n";
{'fraction_non_alphanumeric': 0.17575151841316214, 'fraction_numerical': 0.013684938879065118, 'mean_word_length': 4.113207547169812, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 127, 'https://': 0, 'lorem ipsum': 0, 'www.': 12, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4097874', 'n_tokens_mistral': 4987, 'n_tokens_neox': 4678, 'n_words': 1198}