<commit_before>#Contact <NAME> (<EMAIL>)
use strict;
use Getopt::Long;
use Bio::SeqIO;

BEGIN {
    my $script_dir = $0;
    $script_dir =~ s/(\S+\/)\S+/$1/;
    unshift (@INC, $script_dir);
    require "mapping_conf.pl";
}

my %conf =  %::mapping_conf; # configuration options

# global vars

#Get general options
my $organism   = $conf{'organism'};
my $sptr_swiss = $conf{'sptr_swiss'};
my $out        = $conf{'x_map_out'};

#Get specific options for human/mouse
my $refseq_gnp = $conf{'refseq_gnp'};

#Get specific options for human
my $ens1       = $conf{'ens1'};
my $ens4       = $conf{'ens4'};
my $refeseq_pred = $conf{'refseq_pred'};
my $go           = $conf{'go'}; 

#Get specific options for the mouse
my $mgi_sp     = $conf{'mgi_sp'};
my $mgi_locus  = $conf{'mgi_locus'};

if ((!defined $organism) || (!defined $sptr_swiss) || (!defined $out)) {
    die "\nSome basic options have not been set up, have a look at mapping_conf\nCurrent set up (required options):\norganism: $organism\nsptr_swiss: $sptr_swiss\nx_map: $out\n";
}

my %refseq_map;
my %sp_db;
my %hugo_id;
my %hugo_syn;

open (OUT,">$out") || die "Can't open $out\n";;

#First read the SPTR file in swiss format
print STDERR "Reading SPTR file\n";


my $in  = Bio::SeqIO->new(-file => $sptr_swiss, '-format' =>'swiss');

while ( my $seq = $in->next_seq() ) {

    #For each SP entry, store the information concerning the entry
    my $db;
    my $ac = $seq->accession;
    my $id = $seq->display_id;

    #print STDERR "ID: $id\n";

    my ($displ_id,$tag) = split(/:/,$id);

    if ($tag eq "STANDARD") {
	$db = "SWISS-PROT";
    }
    elsif ($tag eq "PRELIMINARY") {
	$db = "SPTREMBL";
    }
    else {
	die "Try to load unknown SPTR database\n";
    }
    my $un_ac = "$ac:$db";
    my @secs = $seq->get_secondary_accessions;
    my $syns = join(';',@secs);

#These entries correspond to known genes thus they will be flaged as known
    print OUT "$ac\tSPTR\t$ac\t$db\t$displ_id\t$syns\tKNOWN\n";

    $sp_db{$ac} = $db;

    #Then get info about the Xref mapping
    my @dblink = $seq->annotation->each_DBLink;
    
#All of these entries are flaged as XREF, this mean that they have been retrieved using primary DB (SPTR, Refseq)
    foreach my $link(@dblink) {
	if ($link->database eq "EMBL") {
	    print OUT "$ac\tSPTR\t".$link->primary_id."\t".$link->database."\t".$link->primary_id."\t\tXREF\n";

	    my ($protac) = $link->optional_id =~ /^(\w+).\S+/;
	    if ($protac) {
		print OUT "$ac\tSPTR\t".$protac."\tprotein_id\t$protac\t\tXREF\n";
	    }
	}

	if  ($link->database eq "MIM") {
	    print OUT "$ac\tSPTR\t".$link->primary_id."\t".$link->database."\t".$link->primary_id."\t\tXREF\n";
	}

	if  ($link->database eq "PDB") {
	    print STDERR "$ac\tSPTR\t".$link->primary_id."\t".$link->database."\t".$link->primary_id."\t\tXREF\n";
	}

    }


}


#Read the refseq file in gnp format
    print STDERR "Reading REFSEQ File\n";
    
    open (REFSEQ,"$refseq_gnp") || die "Can't open $refseq_gnp\n";
    
    $/ = "\/\/\n";
    
    while (<REFSEQ>) {
	my ($prot_ac) = $_ =~ /ACCESSION\s+(\S+)/;
	my ($dna_ac) = $_ =~ /DBSOURCE    REFSEQ: accession\s+(\w+)/;
	
	$refseq_map{$dna_ac} = $prot_ac; 
	
#Its a curated Refseq, flag it as known
	print OUT "$prot_ac\tRefSeq\t$prot_ac\tRefSeq\t$prot_ac\t\tKNOWN\n";
       
	my ($mim) = $_ =~ /\/db_xref=\"MIM:(\d+)/;
	my ($locus) = $_ =~ /\/db_xref=\"LocusID:(\d*)/;

#XREF entries	
	if ($mim) {
	    print OUT "$prot_ac\tRefSeq\t$mim\tMIM\t$mim\t\tXREF\n";
	}
	
	if ($locus) {
	    print OUT "$prot_ac\tRefSeq\t$locus\tLocusLink\t$locus\t\tXREF\n";
	}
    }
    close (REFSEQ);
    
    $/ = "\n";
    
    


#Get Xref mapping specifically for human
if ($organism eq "human") {

#Read the Hugo files
    print STDERR "Reading Hugo files\n";
    
    open (ENS4,"$ens4") || die "Can't open $ens4\n";;
    
    while (<ENS4>) {
	chomp;
	my @array = split(/\t/,$_);
	my $hgnc = $array[0];
	my $id = $array[1];
	#my $syn1 = $array[2];
	#my $syn2 = $array[3];
	
	my $syn1 = join (';',split (/,\s/,$array[2]));
	my $syn2 = join (';',split (/,\s/,$array[3]));
	
	my $syn = "$syn1;$syn2";
	
	$hugo_id{$hgnc} = $id;
	$hugo_syn{$hgnc} = $syn;
	#print $hugo_syn{$hgnc};
    }
    close (ENS4);
    <commit_msg>parse Refseq file only for mouse and human<commit_after>#Contact <NAME> (<EMAIL>)
use strict;
use Getopt::Long;
use Bio::SeqIO;

BEGIN {
    my $script_dir = $0;
    $script_dir =~ s/(\S+\/)\S+/$1/;
    unshift (@INC, $script_dir);
    require "mapping_conf.pl";
}

my %conf =  %::mapping_conf; # configuration options

# global vars

#Get general options
my $organism   = $conf{'organism'};
my $sptr_swiss = $conf{'sptr_swiss'};
my $out        = $conf{'x_map_out'};

#Get specific options for human/mouse
my $refseq_gnp = $conf{'refseq_gnp'};

#Get specific options for human
my $ens1       = $conf{'ens1'};
my $ens4       = $conf{'ens4'};
my $refeseq_pred = $conf{'refseq_pred'};
my $go           = $conf{'go'}; 

#Get specific options for the mouse
my $mgi_sp     = $conf{'mgi_sp'};
my $mgi_locus  = $conf{'mgi_locus'};

if ((!defined $organism) || (!defined $sptr_swiss) || (!defined $out)) {
    die "\nSome basic options have not been set up, have a look at mapping_conf\nCurrent set up (required options):\norganism: $organism\nsptr_swiss: $sptr_swiss\nx_map: $out\n";
}

my %refseq_map;
my %sp_db;
my %hugo_id;
my %hugo_syn;

open (OUT,">$out") || die "Can't open $out\n";;

#First read the SPTR file in swiss format
print STDERR "Reading SPTR file\n";


my $in  = Bio::SeqIO->new(-file => $sptr_swiss, '-format' =>'swiss');

while ( my $seq = $in->next_seq() ) {

    #For each SP entry, store the information concerning the entry
    my $db;
    my $ac = $seq->accession;
    my $id = $seq->display_id;

    #print STDERR "ID: $id\n";

    my ($displ_id,$tag) = split(/:/,$id);

    if ($tag eq "STANDARD") {
	$db = "SWISS-PROT";
    }
    elsif ($tag eq "PRELIMINARY") {
	$db = "SPTREMBL";
    }
    else {
	die "Try to load unknown SPTR database\n";
    }
    my $un_ac = "$ac:$db";
    my @secs = $seq->get_secondary_accessions;
    my $syns = join(';',@secs);

#These entries correspond to known genes thus they will be flaged as known
    print OUT "$ac\tSPTR\t$ac\t$db\t$displ_id\t$syns\tKNOWN\n";

    $sp_db{$ac} = $db;

    #Then get info about the Xref mapping
    my @dblink = $seq->annotation->each_DBLink;
    
#All of these entries are flaged as XREF, this mean that they have been retrieved using primary DB (SPTR, Refseq)
    foreach my $link(@dblink) {
	if ($link->database eq "EMBL") {
	    print OUT "$ac\tSPTR\t".$link->primary_id."\t".$link->database."\t".$link->primary_id."\t\tXREF\n";

	    my ($protac) = $link->optional_id =~ /^(\w+).\S+/;
	    if ($protac) {
		print OUT "$ac\tSPTR\t".$protac."\tprotein_id\t$protac\t\tXREF\n";
	    }
	}

	if  ($link->database eq "MIM") {
	    print OUT "$ac\tSPTR\t".$link->primary_id."\t".$link->database."\t".$link->primary_id."\t\tXREF\n";
	}

	if  ($link->database eq "PDB") {
	    print STDERR "$ac\tSPTR\t".$link->primary_id."\t".$link->database."\t".$link->primary_id."\t\tXREF\n";
	}

    }


}

if (($organism eq "human") || ($organism eq "mouse")) {
#Read the refseq file in gnp format
    print STDERR "Reading REFSEQ File\n";
    
    open (REFSEQ,"$refseq_gnp") || die "Can't open $refseq_gnp\n";
    
    $/ = "\/\/\n";
    
    while (<REFSEQ>) {
	my ($prot_ac) = $_ =~ /ACCESSION\s+(\S+)/;
	my ($dna_ac) = $_ =~ /DBSOURCE    REFSEQ: accession\s+(\w+)/;
	
	$refseq_map{$dna_ac} = $prot_ac; 
	
#Its a curated Refseq, flag it as known
	print OUT "$prot_ac\tRefSeq\t$prot_ac\tRefSeq\t$prot_ac\t\tKNOWN\n";
       
	my ($mim) = $_ =~ /\/db_xref=\"MIM:(\d+)/;
	my ($locus) = $_ =~ /\/db_xref=\"LocusID:(\d*)/;

#XREF entries	
	if ($mim) {
	    print OUT "$prot_ac\tRefSeq\t$mim\tMIM\t$mim\t\tXREF\n";
	}
	
	if ($locus) {
	    print OUT "$prot_ac\tRefSeq\t$locus\tLocusLink\t$locus\t\tXREF\n";
	}
    }
    close (REFSEQ);
    
    $/ = "\n";
}
    


#Get Xref mapping specifically for human
if ($organism eq "human") {

#Read the Hugo files
    print STDERR "Reading Hugo files\n";
    
    open (ENS4,"$ens4") || die "Can't open $ens4\n";;
    
    while (<ENS4>) {
	chomp;
	my @array = split(/\t/,$_);
	my $hgnc = $array[0];
	my $id = $array[1];
	#my $syn1 = $array[2];
	#my $syn2 = $array[3];
	
	my $syn1 = join (';',split (/,\s/,$array[2]));
	my $syn2 = join (';',split (/,\s/,$array[3]));
	
	my $syn = "$syn1;$syn2";
	
	$hugo_id{$hgnc} = $id;
	$hugo_syn{$hgnc} = $syn;
	#print $hugo_syn{$hgnc};
    }
    close (ENS4);
    
{'fraction_non_alphanumeric': 0.20302072356866877, 'fraction_numerical': 0.0053857862077040155, 'mean_word_length': 3.0949185043144776, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6259927', 'n_tokens_mistral': 3733, 'n_tokens_neox': 3587, 'n_words': 820}