<commit_before>require './web_app'

HOSPITAL_COMPARE_GENERAL_ENDPOINT = "http://data.medicare.gov/resource/v287-28n3.json" # aka https://data.medicare.gov/dataset/Hospital-General-Information/v287-28n3

# MAP ELIGIBLE HOSPITALS (~2k)
puts "Number of hospitals in collection: #{Hospital.count}"
hospitals_without_mapping = Hospital.where("compare.general.provider_number" => nil)
puts "Number of hospitals in collection w/o hospital compare mapping: #{hospitals_without_mapping.count}"

hospitals_without_mapping.each do |h|
  # first, let's look for matches
  full_text_search = "#{h["PROVIDER - ORG NAME"]} #{h["PROVIDER CITY"]} #{h["PROVIDER ZIP 5 CD"]}"
  compare_results = JSON.parse(RestClient.get("#{HOSPITAL_COMPARE_GENERAL_ENDPOINT}?$q=#{URI.escape(full_text_search)}"))
  if compare_results.size == 0
    puts "No mapping found for #{h["PROVIDER - ORG NAME"]} (NPI = #{h["PROVIDER NPI"]})"
  elsif compare_results.size > 1
    puts "More than one match found for #{h["PROVIDER - ORG NAME"]} (NPI = #{h["PROVIDER NPI"]})"
  else # one single match... probably a good sign!
    compare_data = {
      "map_source" => "Socrata full text search w/ name+city+zip",
      "map_last_updated" => Time.now,
      "general" => compare_results.first
    }
    h.update_attribute("compare",compare_data)
    puts "One single match found for #{h["PROVIDER - ORG NAME"]} (NPI = #{h["PROVIDER NPI"]})!"
  end

end<commit_msg>making it easier to debug<commit_after>require './web_app'

HOSPITAL_COMPARE_GENERAL_ENDPOINT = "http://data.medicare.gov/resource/v287-28n3.json" # aka https://data.medicare.gov/dataset/Hospital-General-Information/v287-28n3

# MAP ELIGIBLE HOSPITALS (~2k)
puts "Number of hospitals in collection: #{Hospital.count}"
hospitals_without_mapping = Hospital.where("compare.general.provider_number" => nil)
puts "Number of hospitals in collection w/o hospital compare mapping: #{hospitals_without_mapping.count}"

hospitals_without_mapping.each do |h|
  # first, let's look for matches
  full_text_search = "#{h["PROVIDER - ORG NAME"]} #{h["PROVIDER CITY"]} #{h["PROVIDER ZIP 5 CD"]}"
  compare_results = JSON.parse(RestClient.get("#{HOSPITAL_COMPARE_GENERAL_ENDPOINT}?$q=#{URI.escape(full_text_search)}"))
  if compare_results.size == 0
    puts "No mapping found for #{h["PROVIDER - ORG NAME"]} (NPI = #{h["PROVIDER NPI"]})"
  elsif compare_results.size > 1
    puts "More than one match found for #{h["PROVIDER - ORG NAME"]} (NPI = #{h["PROVIDER NPI"]})"
  else # one single match... probably a good sign!
    puts "One single match found for #{h["PROVIDER - ORG NAME"]} (NPI = #{h["PROVIDER NPI"]})!"
    compare_data = {
      "map_source" => "Socrata full text search w/ name+city+zip",
      "map_last_updated" => Time.now,
      "general" => compare_results.first
    }
    h.update_attribute("compare",compare_data)
  end

end
{'fraction_non_alphanumeric': 0.14947368421052631, 'fraction_numerical': 0.011228070175438596, 'mean_word_length': 4.830265848670757, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2730133', 'n_tokens_mistral': 1033, 'n_tokens_neox': 1000, 'n_words': 235}