<commit_before>use strict;
use warnings;

package ImportUtils;

use Exporter;

our @ISA = ('Exporter');

our @EXPORT_OK = qw(dumpSQL debug create_and_load load);

our $TMP_DIR = "/ecs2/scratch5/ensembl/mcvicker/dbSNP";
our $TMP_FILE = 'tabledump.txt';

# successive dumping and loading of tables is typical for this process
# dump does effectively a select into outfile without server file system access
sub dumpSQL {
  my $db  = shift;
  my $sql = shift;

  local *FH;

  open FH, ">$TMP_DIR/$TMP_FILE";

  my $sth = $db->prepare( $sql, { mysql_use_result => 1 });
  $sth->execute();
  my $first;
  while ( my $aref = $sth->fetchrow_arrayref() ) {
    my @a = map {defined($_) ? $_ : '\N'} @$aref;<commit_msg>updated load to accept bind variables<commit_after>use strict;
use warnings;

package ImportUtils;

use Exporter;

our @ISA = ('Exporter');

our @EXPORT_OK = qw(dumpSQL debug create_and_load load);

our $TMP_DIR = "/ecs2/scratch5/ensembl/mcvicker/dbSNP";
our $TMP_FILE = 'tabledump.txt';

# successive dumping and loading of tables is typical for this process
# dump does effectively a select into outfile without server file system access
sub dumpSQL {
  my $db  = shift;
  my $sql = shift;
  my @vars = shift; #array containing all the bind variables
  local *FH;

  open FH, ">$TMP_DIR/$TMP_FILE";

  my $sth = $db->prepare( $sql, { mysql_use_result => 1 });
  $sth->execute(@vars);
  my $first;
  while ( my $aref = $sth->fetchrow_arrayref() ) {
    my @a = map {defined($_) ? $_ : '\N'} @$aref;
{'fraction_non_alphanumeric': 0.13850231941683233, 'fraction_numerical': 0.003976143141153081, 'mean_word_length': 4.11864406779661, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3803819', 'n_tokens_mistral': 552, 'n_tokens_neox': 535, 'n_words': 169}