<commit_before># compare existing ensembl mappings with those generated by new xref mapper

# files should be in format
# xref_accession ensembl_type ensembl_id

# to dump this from an ensembl datbase use something like
# mysql -u ensro -h ecs2 -P 3365 -e "select x.dbprimary_acc, ox.ensembl_object_type, ox.ensembl_id from xref x, object_xref ox, external_db edb where edb.db_name like 'Uniprot/SWISSPROT' and x.external_db_id=edb.external_db_id and x.xref_id=ox.xref_id order by x.xref_id" homo_sapiens_core_25_34e > ensembl_swissprot_translations.txt

use strict;

use Getopt::Long;

my ( $old, $new, $pass, $port, $dbname  );

GetOptions( "old=s", \$old,
	    "new=s", \$new);

if( !($old && $new) ) {
  usage();
  exit(1);
}

# read files into xref/object hashes
# TODO - more than one mapping?

my ($new_x2e_r, $new_e2x_r) = read_mappings($new);
my ($old_x2e_r, $old_e2x_r) = read_mappings($old);

#my %new_x2e = %$new_x2e_r;
#my %old_x2e = %$old_x2e_r;
#my %new_e2x = %$new_e2x_r;
#my %old_e2x = %$old_e2x_r;

compare($old_x2e_r, $new_x2e_r, "xref");
compare($old_e2x_r, $new_e2x_r, "ensembl_object");

# ----------------------------------------
# Compare mappings keyed on xref

sub compare {

  my ($oldr, $newr, $desc) = @_;

  open(NEW_ONLY, ">${desc}_new_only.txt");
  open(OLD_ONLY, ">${desc}_old_only.txt");

  my %new = %$newr;
  my %old = %$oldr;

  my ($matched, $mismatched, $new_only, $old_only, $total);

  foreach my $key (keys %new) {

    # if a mapping exists, look for any matches
    my $found = 0;
    if (exists $old{$key}) {
      foreach my $old_value (@{$old{$key}}) {
	foreach my $new_value (@{$new{$key}}) {
	  if ($old_value eq $new_value) {
	    $found = 1;
	  }
	}
      }
      if ($found) {
	$matched++;
      } else {
	$mismatched++;
      }
      $total++;

    } else {
      $new_only++;
      print NEW_ONLY join("\t", split(/\./, $key)) . "\n";
    }

  }

  foreach my $key (keys %old) {

    if (!exists $new{$key}) {
      $old_only++;
      print OLD_ONLY join("\t", split(/\./, $key)) . "\n";
    }

  }

  close(NEW_ONLY);
  close(OLD_ONLY);
  close(MISMATCHED);

  print "\nCompared " . scalar(keys %new) . " new xref mappings with " . scalar(keys %old) . " existing ensembl mappings\n";

  print "\nComparing keyed on $desc:\n\n";

  my $fmt = "%-30s\t%6d\t%5s%%\n";
  printf $fmt, "Mapped with same result:", $matched, pc($matched, $total);<commit_msg>Tidied up to use hashrefs rather than passing whole hashes around.<commit_after># compare existing ensembl mappings with those generated by new xref mapper

# files should be in format
# xref_accession ensembl_type ensembl_id

# to dump this from an ensembl datbase use something like
# mysql -u ensro -h ecs2 -P 3365 -e "select x.dbprimary_acc, ox.ensembl_object_type, ox.ensembl_id from xref x, object_xref ox, external_db edb where edb.db_name like 'Uniprot/SWISSPROT' and x.external_db_id=edb.external_db_id and x.xref_id=ox.xref_id order by x.xref_id" homo_sapiens_core_25_34e > ensembl_swissprot_translations.txt

use strict;

use Getopt::Long;

my ( $old, $new, $pass, $port, $dbname  );

GetOptions( "old=s", \$old,
	    "new=s", \$new);

if( !($old && $new) ) {
  usage();
  exit(1);
}

# read files into xref/object hashes
# TODO - more than one mapping?

my ($new_x2e_r, $new_e2x_r) = read_mappings($new);
my ($old_x2e_r, $old_e2x_r) = read_mappings($old);

compare($old_x2e_r, $new_x2e_r, "xref");
compare($old_e2x_r, $new_e2x_r, "ensembl_object");

# ----------------------------------------
# Compare mappings keyed on xref

sub compare {

  my ($old, $new, $desc) = @_;

  open(NEW_ONLY, ">${desc}_new_only.txt");
  open(OLD_ONLY, ">${desc}_old_only.txt");

  my ($matched, $mismatched, $new_only, $old_only, $total);

  foreach my $key (keys %$new) {

    # if a mapping exists, look for any matches
    my $found = 0;
    if (exists $old->{$key}) {
      foreach my $old_value (@{$old->{$key}}) {
	foreach my $new_value (@{$new->{$key}}) {
	  if ($old_value eq $new_value) {
	    $found = 1;
	  }
	}
      }
      if ($found) {
	$matched++;
      } else {
	$mismatched++;
      }
      $total++;

    } else {
      $new_only++;
      print NEW_ONLY join("\t", split(/\./, $key)) . "\n";
    }

  }

  foreach my $key (keys %$old) {

    if (!exists $new->{$key}) {
      $old_only++;
      print OLD_ONLY join("\t", split(/\./, $key)) . "\n";
    }

  }

  close(NEW_ONLY);
  close(OLD_ONLY);
  close(MISMATCHED);

  print "\nCompared " . scalar(keys %$new) . " new xref mappings with " . scalar(keys %$old) . " existing ensembl mappings\n";

  print "\nComparing keyed on $desc:\n\n";

  my $fmt = "%-30s\t%6d\t%5s%%\n";
  printf $fmt, "Mapped with same result:", $matched, pc($matched, $total);
{'fraction_non_alphanumeric': 0.18414918414918416, 'fraction_numerical': 0.01186692095783005, 'mean_word_length': 3.236983842010772, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1551453', 'n_tokens_mistral': 1946, 'n_tokens_neox': 1867, 'n_words': 471}