<commit_before>package XrefParser::BaseParser;

use strict;

use Carp;
use DBI;
use Digest::MD5 qw(md5_hex);
use Getopt::Long;
use POSIX qw(strftime);

use File::Basename;
use File::Path;
use IO::File;

use Bio::EnsEMBL::Utils::Exception;

my $base_dir = ".";

my $add_xref_sth = undef;
my $add_direct_xref_sth = undef;
my $add_dependent_xref_sth = undef;
my $get_xref_sth = undef;
my $add_synonym_sth = undef;

my $dbi;
my %dependent_sources;
my %taxonomy2species_id;
my %name2species_id;

my (
    $host,         $port,    $dbname,
    $user,         $pass,    $create,
    $release,      $cleanup, $deletedownloaded,
    $skipdownload, $drop_db, $checkdownload,
    $dl_path,      $unzip
);

# --------------------------------------------------------------------------------
# Get info about files to be parsed from the database

sub run
{
    my $self = shift;

    (
        $host,           $port,             $dbname,
        $user,           $pass,             my $speciesr,
        my $sourcesr,    $skipdownload,     $checkdownload,
        $create,         $release,          $cleanup,
        $drop_db,        $deletedownloaded, $dl_path,
        my $notsourcesr, $unzip
    ) = @_;

  $base_dir = $dl_path if $dl_path;

  my @species = @$speciesr;
  my @sources = @$sourcesr;
  my @notsources = @$notsourcesr;

  my $sql_dir = dirname($0);

  create($host, $port, $user, $pass, $dbname, $sql_dir."/" , $drop_db ) if ($create);

  my $dbi = dbi();

  # validate species names
  my @species_ids = validate_species(@species);

  # validate source names
  exit(1) if (!validate_sources(@sources));
  exit(1) if (!validate_sources(@notsources));

  # build SQL
  my $species_sql = "";
  if (@species_ids) {
    $species_sql .= " AND su.species_id IN (";
    for (my $i = 0; $i < @species_ids; $i++ ) {
      $species_sql .= "," if ($i ne 0);
      $species_sql .= $species_ids[$i];
    }
    $species_sql .= ") ";
  }

  my $source_sql = "";
  if (@sources) {
    $source_sql .= " AND LOWER(s.name) IN (";
    for (my $i = 0; $i < @sources; $i++ ) {
      $source_sql .= "," if ($i ne 0);
      $source_sql .= "\'" . lc($sources[$i]) . "\'";
    }
    $source_sql .= ") ";
  }

  if (@notsources) {
    $source_sql .= " AND LOWER(s.name) NOT IN (";
    for (my $i = 0; $i < @notsources; $i++ ) {
      $source_sql .= "," if ($i ne 0);
      $source_sql .= "\'" . lc($notsources[$i]) . "\'";
    }
    $source_sql .= ") ";
  }

  my $sql =
    "SELECT distinct(s.source_id), su.source_url_id, s.name, su.url, su.checksum, su.parser, su.species_id " .
      "FROM source s, source_url su, species sp " .
	"WHERE s.download='Y' AND su.source_id=s.source_id " .
          "AND su.species_id=sp.species_id " .
	    $source_sql . $species_sql .
	      "ORDER BY s.ordered";
  # print $sql . "\n";

  my $sth = $dbi->prepare($sql);
  $sth->execute();

  my ( $source_id, $source_url_id, $name, $url, $checksum, $parser,
      $species_id);

  $sth->bind_columns( \$source_id, \$source_url_id, \$name, \$url,
      \$checksum, \$parser, \$species_id);

  my $last_type = "";
  my $dir;
  my %summary=();
  while (my @row = $sth->fetchrow_array()) {
    print '-' x 4, "{ $name }", '-' x ( 72 - length($name) ), "\n";

    # Download each source into the appropriate directory for parsing later
    # or call the appropriate database parser if appropriate
    # Also delete previous working directory if we're starting a new source type

    # can have more than one file

    my @files = split(/\s+/,$url);

    my $parse = 0;
    my $empty = 0;
    my $file_cs = "";
    my $cs;
    my $type = $name;
    my @new_file=();
    $dir = $base_dir . "/" . sanitise($type);
    my $dsn;
    $summary{$parser} = 0;
    ##
    # for summary purposes if 0 is returned then it is successful.
    #                         1 is returned then if failed.
    #                     undef/nothing  is returned the we do not know
    ##
    foreach my $urls (@files){

      # Database parsing
      if ($urls =~ /^mysql:/i) {
	$dsn = $urls;
	print "Parsing $dsn with $parser\n";
        eval "require XrefParser::$parser";
        my $new = "XrefParser::$parser"->new();
        if($new->run($dsn, $source_id, $species_id, $name, undef)){
	  $summary{$parser}++;
	}
	next;
      }

      # Local files need to be dealt with specially; assume they are specified as
      # LOCAL:location/of/file/relative/to/xref_mapper
      my ($file) = $urls =~ /.*\/(.*)/;
      if ($urls =~ /^LOCAL:(.*)/i) {
	my $local_file = $1;
        if ( !defined( $cs = md5sum($local_file) ) ) {
            print "Download '$local_file'\n";
            $summary{$parser}++;
        }
	else {
          $file_cs .= ':' . $cs;
	  if (!defined $checksum || index($checksum, $file_cs) == -1) {
	    print "Checksum for '$file' does not match, parsing\n";
	    print "Parsing local file '$local_file' with $parser\n";
	    eval "require XrefParser::$parser";
	    my $new = "XrefParser::$parser"->new();
	    if($new->run($source_id, $species_id, $local_file)){
	      $summary{$parser}++;
	    }
	    else{
                update_source( $dbi, $source_url_id, $file_cs,
                    $local_file );
	    }
	  }
	  else{
	    print "Ignoring '$file' as checksums match\n";
	  }
	}
	next;<commit_msg>Allow us to specify local files with 'file://' (which is standard) as well as with 'LOCAL:' (which isn't).<commit_after>package XrefParser::BaseParser;

use strict;

use Carp;
use DBI;
use Digest::MD5 qw(md5_hex);
use Getopt::Long;
use POSIX qw(strftime);

use File::Basename;
use File::Path;
use IO::File;

use Bio::EnsEMBL::Utils::Exception;

my $base_dir = ".";

my $add_xref_sth = undef;
my $add_direct_xref_sth = undef;
my $add_dependent_xref_sth = undef;
my $get_xref_sth = undef;
my $add_synonym_sth = undef;

my $dbi;
my %dependent_sources;
my %taxonomy2species_id;
my %name2species_id;

my (
    $host,         $port,    $dbname,
    $user,         $pass,    $create,
    $release,      $cleanup, $deletedownloaded,
    $skipdownload, $drop_db, $checkdownload,
    $dl_path,      $unzip
);

# --------------------------------------------------------------------------------
# Get info about files to be parsed from the database

sub run
{
    my $self = shift;

    (
        $host,           $port,             $dbname,
        $user,           $pass,             my $speciesr,
        my $sourcesr,    $skipdownload,     $checkdownload,
        $create,         $release,          $cleanup,
        $drop_db,        $deletedownloaded, $dl_path,
        my $notsourcesr, $unzip
    ) = @_;

  $base_dir = $dl_path if $dl_path;

  my @species = @$speciesr;
  my @sources = @$sourcesr;
  my @notsources = @$notsourcesr;

  my $sql_dir = dirname($0);

  create($host, $port, $user, $pass, $dbname, $sql_dir."/" , $drop_db ) if ($create);

  my $dbi = dbi();

  # validate species names
  my @species_ids = validate_species(@species);

  # validate source names
  exit(1) if (!validate_sources(@sources));
  exit(1) if (!validate_sources(@notsources));

  # build SQL
  my $species_sql = "";
  if (@species_ids) {
    $species_sql .= " AND su.species_id IN (";
    for (my $i = 0; $i < @species_ids; $i++ ) {
      $species_sql .= "," if ($i ne 0);
      $species_sql .= $species_ids[$i];
    }
    $species_sql .= ") ";
  }

  my $source_sql = "";
  if (@sources) {
    $source_sql .= " AND LOWER(s.name) IN (";
    for (my $i = 0; $i < @sources; $i++ ) {
      $source_sql .= "," if ($i ne 0);
      $source_sql .= "\'" . lc($sources[$i]) . "\'";
    }
    $source_sql .= ") ";
  }

  if (@notsources) {
    $source_sql .= " AND LOWER(s.name) NOT IN (";
    for (my $i = 0; $i < @notsources; $i++ ) {
      $source_sql .= "," if ($i ne 0);
      $source_sql .= "\'" . lc($notsources[$i]) . "\'";
    }
    $source_sql .= ") ";
  }

  my $sql =
    "SELECT distinct(s.source_id), su.source_url_id, s.name, su.url, su.checksum, su.parser, su.species_id " .
      "FROM source s, source_url su, species sp " .
	"WHERE s.download='Y' AND su.source_id=s.source_id " .
          "AND su.species_id=sp.species_id " .
	    $source_sql . $species_sql .
	      "ORDER BY s.ordered";
  # print $sql . "\n";

  my $sth = $dbi->prepare($sql);
  $sth->execute();

  my ( $source_id, $source_url_id, $name, $url, $checksum, $parser,
      $species_id);

  $sth->bind_columns( \$source_id, \$source_url_id, \$name, \$url,
      \$checksum, \$parser, \$species_id);

  my $last_type = "";
  my $dir;
  my %summary=();
  while (my @row = $sth->fetchrow_array()) {
    print '-' x 4, "{ $name }", '-' x ( 72 - length($name) ), "\n";

    # Download each source into the appropriate directory for parsing later
    # or call the appropriate database parser if appropriate
    # Also delete previous working directory if we're starting a new source type

    # can have more than one file

    my @files = split(/\s+/,$url);

    my $parse = 0;
    my $empty = 0;
    my $file_cs = "";
    my $cs;
    my $type = $name;
    my @new_file=();
    $dir = $base_dir . "/" . sanitise($type);
    my $dsn;
    $summary{$parser} = 0;
    ##
    # for summary purposes if 0 is returned then it is successful.
    #                         1 is returned then if failed.
    #                     undef/nothing  is returned the we do not know
    ##
    foreach my $urls (@files){

      # Database parsing
      if ($urls =~ /^mysql:/i) {
	$dsn = $urls;
	print "Parsing $dsn with $parser\n";
        eval "require XrefParser::$parser";
        my $new = "XrefParser::$parser"->new();
        if($new->run($dsn, $source_id, $species_id, $name, undef)){
	  $summary{$parser}++;
	}
	next;
      }

      # Local files need to be dealt with specially; assume they are specified as
      # LOCAL:location/of/file/relative/to/xref_mapper

      # FIXME?: Temporary (?) fix to allow us to understand that
      #         'file://' is the same as 'LOCAL:'.
      $urls =~ s#^file://#LOCAL:#;

      my ($file) = $urls =~ /.*\/(.*)/;
      if ($urls =~ /^LOCAL:(.*)/i) {
	my $local_file = $1;
        if ( !defined( $cs = md5sum($local_file) ) ) {
            print "Download '$local_file'\n";
            $summary{$parser}++;
        }
	else {
          $file_cs .= ':' . $cs;
	  if (!defined $checksum || index($checksum, $file_cs) == -1) {
	    print "Checksum for '$file' does not match, parsing\n";
	    print "Parsing local file '$local_file' with $parser\n";
	    eval "require XrefParser::$parser";
	    my $new = "XrefParser::$parser"->new();
	    if($new->run($source_id, $species_id, $local_file)){
	      $summary{$parser}++;
	    }
	    else{
                update_source( $dbi, $source_url_id, $file_cs,
                    $local_file );
	    }
	  }
	  else{
	    print "Ignoring '$file' as checksums match\n";
	  }
	}
	next;
{'fraction_non_alphanumeric': 0.18321886021904585, 'fraction_numerical': 0.004455169853350659, 'mean_word_length': 2.3102918586789554, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7385466', 'n_tokens_mistral': 3995, 'n_tokens_neox': 3793, 'n_words': 983}