<commit_before>namespace :hcahps do

  desc "Fetch from Socrata API for https://data.medicare.gov/dataset/Survey-of-Patients-Hospital-Experiences-HCAHPS-/rj76-22dk"
  task :ingest do
    SOCRATA_APP_TOKEN = "<KEY>"
    SOCRATA_ENDPOINT = "http://data.medicare.gov/resource/rj76-22dk.json"
    SOCRATA_ENDPOINT += "?$$app_token=#{SOCRATA_APP_TOKEN}"

    # MAP ELIGIBLE HOSPITALS (~2k)
    puts "Number of hospitals in collection: #{Hospital.count}"
    hospitals_without_hcahps = Hospital.where("hcahps.provider_number" => nil)
    puts "Number of hospitals in collection w/o HCAHPS: #{hospitals_without_hcahps.count}"

    hospitals_without_hcahps.each do |h|
      ensure_proper_ccn_format(h)
      request_url = "#{SOCRATA_ENDPOINT}&provider_number=#{h["PROVIDER CCN"]}"
      hcahps_results = JSON.parse(RestClient.get(request_url))
      if hcahps_results.size == 0
        puts "No hcahps data found for #{h["PROVIDER - ORG NAME"]} (CCN = #{h["PROVIDER CCN"]})"
      elsif hcahps_results.size > 1
        puts "More than one match found for #{h["PROVIDER - ORG NAME"]} (CCN = #{h["PROVIDER CCN"]})"
      else
        puts "Found HCAHPS data for #{h["PROVIDER - ORG NAME"]} (CCN = #{h["PROVIDER CCN"]})!"
        hcahps_data = {
          "_source" => request_url,
          "_updated_at" => Time.now,
        }.merge(hcahps_results.first)
        h.update_attribute("hcahps",hcahps_data)
      end
    end
  end

end
<commit_msg>using socrata HTTP header instead of URL param for API token<commit_after>namespace :hcahps do

  desc "Fetch from Socrata API for https://data.medicare.gov/dataset/Survey-of-Patients-Hospital-Experiences-HCAHPS-/rj76-22dk"
  task :ingest do
    SOCRATA_ENDPOINT = "http://data.medicare.gov/resource/rj76-22dk.json"

    # MAP ELIGIBLE HOSPITALS (~2k)
    puts "Number of hospitals in collection: #{Hospital.count}"
    hospitals_without_hcahps = Hospital.where("hcahps.provider_number" => nil)
    puts "Number of hospitals in collection w/o HCAHPS: #{hospitals_without_hcahps.count}"

    hospitals_without_hcahps.each do |h|
      ensure_proper_ccn_format(h)
      request_url = "#{SOCRATA_ENDPOINT}?provider_number=#{h["PROVIDER CCN"]}"
      hcahps_results = JSON.parse(RestClient.get(request_url, {"X-App-Token" => SOCRATA_APP_TOKEN}))
      if hcahps_results.size == 0
        puts "No hcahps data found for #{h["PROVIDER - ORG NAME"]} (CCN = #{h["PROVIDER CCN"]})"
      elsif hcahps_results.size > 1
        puts "More than one match found for #{h["PROVIDER - ORG NAME"]} (CCN = #{h["PROVIDER CCN"]})"
      else
        puts "Found HCAHPS data for #{h["PROVIDER - ORG NAME"]} (CCN = #{h["PROVIDER CCN"]})!"
        hcahps_data = {
          "_source" => request_url,
          "_updated_at" => Time.now,
        }.merge(hcahps_results.first)
        h.update_attribute("hcahps",hcahps_data)
      end
    end
  end

end

{'fraction_non_alphanumeric': 0.1335891175444716, 'fraction_numerical': 0.0076735263341471925, 'mean_word_length': 3.467289719626168, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6573535', 'n_tokens_mistral': 1078, 'n_tokens_neox': 1059, 'n_words': 218}