<commit_before>#!/usr/bin/env ruby

Encoding.default_external = Encoding::UTF_8
Encoding.default_internal = Encoding::UTF_8

require 'csv'

internet_archive_csv, hathifile_tsv = ARGV

identifiers = {}
ia_volumes = {}
ia_date = {}
ia_year = {}
ht_year = {}
hathi_volumes = {}
ht_ia_match_scores = {}

def check_volumes(ht_vol, ia_vol)
  if ht_vol == ia_vol
    return true
  end

  ht_vol_number_match = /(\d+)/.match(ht_vol)
  ia_vol_number_match = /(\d+)/.match(ia_vol)

  # if we don't get a number from both vol strings, just say it's a match
  if ht_vol_number_match.nil? || ht_vol_number_match.captures.empty? || ia_vol_number_match.nil? || ia_vol_number_match.captures.empty?
    return true
  else
    # if we get numbers from both vol strings, only say it's a match if they're the same
    ht_vol_number = ht_vol_number_match.captures.first
    ia_vol_number = ia_vol_number_match.captures.first
    if ht_vol_number == ia_vol_number
      return true
    end
  end

  return false
end

def check_published(ht_pub, ia_pub)
  return ht_pub == ia_pub
end

def check_standard_identifiers(ht_ia_match_scores, standard_identifiers, ht_identifier, identifier_string, standard_identifier_key)
  if !identifier_string.nil? && !identifier_string.empty?
    identifier_string.split(',').each do |identifier_to_check|
      if standard_identifiers[standard_identifier_key].has_key?(identifier_to_check)
        standard_identifiers[standard_identifier_key][identifier_to_check].each do |ia_identifier|
          ht_ia_match_scores[ht_identifier] ||= {}
          ht_ia_match_scores[ht_identifier][ia_identifier] ||= 0
          ht_ia_match_scores[ht_identifier][ia_identifier] += 1
        end<commit_msg>Fix accidental method rename<commit_after>#!/usr/bin/env ruby

Encoding.default_external = Encoding::UTF_8
Encoding.default_internal = Encoding::UTF_8

require 'csv'

internet_archive_csv, hathifile_tsv = ARGV

identifiers = {}
ia_volumes = {}
ia_date = {}
ia_year = {}
ht_year = {}
hathi_volumes = {}
ht_ia_match_scores = {}

def check_volumes(ht_vol, ia_vol)
  if ht_vol == ia_vol
    return true
  end

  ht_vol_number_match = /(\d+)/.match(ht_vol)
  ia_vol_number_match = /(\d+)/.match(ia_vol)

  # if we don't get a number from both vol strings, just say it's a match
  if ht_vol_number_match.nil? || ht_vol_number_match.captures.empty? || ia_vol_number_match.nil? || ia_vol_number_match.captures.empty?
    return true
  else
    # if we get numbers from both vol strings, only say it's a match if they're the same
    ht_vol_number = ht_vol_number_match.captures.first
    ia_vol_number = ia_vol_number_match.captures.first
    if ht_vol_number == ia_vol_number
      return true
    end
  end

  return false
end

def check_published(ht_pub, ia_pub)
  return ht_pub == ia_pub
end

def check_identifiers(ht_ia_match_scores, standard_identifiers, ht_identifier, identifier_string, standard_identifier_key)
  if !identifier_string.nil? && !identifier_string.empty?
    identifier_string.split(',').each do |identifier_to_check|
      if standard_identifiers[standard_identifier_key].has_key?(identifier_to_check)
        standard_identifiers[standard_identifier_key][identifier_to_check].each do |ia_identifier|
          ht_ia_match_scores[ht_identifier] ||= {}
          ht_ia_match_scores[ht_identifier][ia_identifier] ||= 0
          ht_ia_match_scores[ht_identifier][ia_identifier] += 1
        end
{'fraction_non_alphanumeric': 0.09325513196480939, 'fraction_numerical': 0.002346041055718475, 'mean_word_length': 4.666112956810632, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2919370', 'n_tokens_mistral': 1246, 'n_tokens_neox': 1189, 'n_words': 232}