<commit_before>#!/usr/bin/perl
# translates data from ensembl database to intermine items XML file

BEGIN {
    my $base = ( $0 =~ m:(.*)/.*: )[0];
    unshift( @INC, 
        map( {$base . $_} 
            '/../../../intermine/perl/InterMine-Util/lib',
            '/../../../intermine/perl/InterMine-Item/lib',
        ),
    );
}

use strict;
use warnings;

use Getopt::Std;

use InterMine::Item::Document;
use InterMine::Model;
use Bio::EnsEMBL::DBSQL::DBAdaptor;
use InterMine::Util qw(parse_properties_file);
use Log::Handler;
use Digest::MD5 qw(md5);

my $script_dir = ( $0 =~ m:(.*)/.*: )[0];

our ( $opt_r, $opt_l );
getopt('rl');

if ( @ARGV != 3 ) {
    die "usage: [-r release] [-l logfile] mine_name taxon_id data_destination\n"
      . "eg.    [-r preview] [-l out.log] flymine   9606     /shared/data/ensembl/current\n";
}

# vars from the command line
my ( $mine_name, $taxon_ids, $data_destination ) = @ARGV;

# just for logging purposes
my $start_time = time();

my $logfile = $opt_l;
my $log = Log::Handler->new();
if ($logfile) {
    $log->add(
        file => {
            filename => $logfile,
            mode => 'append',
            newline => 1,
        },
    );
} else {
    $log->add(
        screen => {
            log_to => "STDERR",
        },
    );
}

$log->info("Running $0");

# Set-up the intermine item-creating apparatus
my $release = ($opt_r) ? '.' . $opt_r : '';
my $model_file =
  $script_dir . "/../../../$mine_name/dbmodel/build/model/genomic_model.xml";
my $properties_file = "$ENV{HOME}/.intermine/$mine_name.properties" . $release;

for ( $model_file, $properties_file ) {
    $log->die("Cannot read $_")unless -r;
}

my $model = InterMine::Model->new( file => $model_file );
my $properties = parse_properties_file($properties_file);

# init vars which need to be globally accessible
#my (@items,
my (%item_store);
my %organisms  = parse_orgs($taxon_ids);
my $datasource = 'Ensembl';
my ( $org_item, $dataset_item, $datasource_item, %genesncbis, %ncbisgenes )
  ;    # of these, only the two hashes should grow,

# getting to about 500kb for every 200mb of xml

# config file to tell us which chromosomes to bother with for which organisms
# the default is _all_ chromosomes
my $config_file =
  $script_dir . '/../../sources/ensembl/resources/ensembl_config.properties';
%organisms = parse_config( $config_file, %organisms );
my $item_factory;

my $has_been_written =
  {};    # state variables written as closures for compatibility with v5.8
my $total_written;
my $total_made;

sub writeout {
    my $item            = shift;
    my $uniq            = $item->{':uniq'};
    my $implements      = $item->{':implements'};
    my $this_id         = $item->{'id'};
    my $already_written = $has_been_written->{$implements}->{$uniq};

    if ($already_written) {
        &announce_duplicate( $this_id, $implements, $uniq, $already_written )
          unless ( $already_written == $this_id );

        # ie. if it isn't _this_ item which we wrote
        return;
    }
    else {
        $item_factory->write($item);
        $has_been_written->{$implements}->{$uniq} = $this_id;
        $total_written++;
    }<commit_msg>changed relative path<commit_after>#!/usr/bin/perl
# translates data from ensembl database to intermine items XML file

BEGIN {
    my $base = ( $0 =~ m:(.*)/.*: )[0];
    unshift( @INC, 
        map( {$base . $_} 
            '/../../../../intermine/perl/InterMine-Util/lib',
            '/../../../../intermine/perl/InterMine-Item/lib',
            '/../../../../intermine/perl/InterMine-Model/lib',
        ),
    );
}

use strict;
use warnings;

use Getopt::Std;

use InterMine::Item::Document;
use InterMine::Model;
use Bio::EnsEMBL::DBSQL::DBAdaptor;
use InterMine::Util qw(parse_properties_file);
use Log::Handler;
use Digest::MD5 qw(md5);

my $script_dir = ( $0 =~ m:(.*)/.*: )[0];

our ( $opt_r, $opt_l );
getopt('rl');

if ( @ARGV != 3 ) {
    die "usage: [-r release] [-l logfile] mine_name taxon_id data_destination\n"
      . "eg.    [-r preview] [-l out.log] flymine   9606     /shared/data/ensembl/current\n";
}

# vars from the command line
my ( $mine_name, $taxon_ids, $data_destination ) = @ARGV;

# just for logging purposes
my $start_time = time();

my $logfile = $opt_l;
my $log = Log::Handler->new();
if ($logfile) {
    $log->add(
        file => {
            filename => $logfile,
            mode => 'append',
            newline => 1,
        },
    );
} else {
    $log->add(
        screen => {
            log_to => "STDERR",
        },
    );
}

$log->info("Running $0");

# Set-up the intermine item-creating apparatus
my $release = ($opt_r) ? '.' . $opt_r : '';
my $model_file =
  $script_dir . "/../../../../$mine_name/dbmodel/build/model/genomic_model.xml";
my $properties_file = "$ENV{HOME}/.intermine/$mine_name.properties" . $release;

for ( $model_file, $properties_file ) {
    $log->die("Cannot read $_")unless -r;
}

my $model = InterMine::Model->new( file => $model_file );
my $properties = parse_properties_file($properties_file);

# init vars which need to be globally accessible
#my (@items,
my (%item_store);
my %organisms  = parse_orgs($taxon_ids);
my $datasource = 'Ensembl';
my ( $org_item, $dataset_item, $datasource_item, %genesncbis, %ncbisgenes )
  ;    # of these, only the two hashes should grow,

# getting to about 500kb for every 200mb of xml

# config file to tell us which chromosomes to bother with for which organisms
# the default is _all_ chromosomes
my $config_file =
  $script_dir . '/../../../sources/ensembl/resources/ensembl_config.properties';
%organisms = parse_config( $config_file, %organisms );
my $item_factory;

my $has_been_written =
  {};    # state variables written as closures for compatibility with v5.8
my $total_written;
my $total_made;

sub writeout {
    my $item            = shift;
    my $uniq            = $item->{':uniq'};
    my $implements      = $item->{':implements'};
    my $this_id         = $item->{'id'};
    my $already_written = $has_been_written->{$implements}->{$uniq};

    if ($already_written) {
        &announce_duplicate( $this_id, $implements, $uniq, $already_written )
          unless ( $already_written == $this_id );

        # ie. if it isn't _this_ item which we wrote
        return;
    }
    else {
        $item_factory->write($item);
        $has_been_written->{$implements}->{$uniq} = $this_id;
        $total_written++;
    }
{'fraction_non_alphanumeric': 0.17017297802711548, 'fraction_numerical': 0.0065451145395044414, 'mean_word_length': 2.8592904389657248, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5510730', 'n_tokens_mistral': 2311, 'n_tokens_neox': 2206, 'n_words': 543}