<commit_before>module ClinicalTrials
  class Updater
    attr_reader :params, :load_event, :client, :study_counts, :study_filter

    def initialize(params={})
      @params=params
      type=(params[:event_type] ? params[:event_type] : 'incremental')
      if params[:restart]
        puts("Restarting the load...")
        # don't allow filtering by nct ID unless it's a restart because filtering is done so that multiple loads can run simultaneously
        # if multiple jobs running for initial full load - they would step on each ther when study_xml_records table gets truncated
        @study_filter=@params[:study_filter]
        record_type="Restart #{@study_filter}"
      else
        puts("Starting the #{type} load...")
        record_type=type
      end
      @client = ClinicalTrials::Client.new
      @load_event = LoadEvent.create({:event_type=>record_type,:status=>'running',:description=>'',:problems=>''})
      @study_counts={:should_add=>0,:should_change=>0,:processed=>0,:count_down=>0}
      self
    end

    def run
      ActiveRecord::Base.logger=nil
      if study_filter
        @client.populate_studies(study_filter)
        load_event.complete({:new_studies=> Study.count})
      else
        case params[:event_type]
        when 'full'
          full
        when 'finalize'
          finalize_full_load
        else
          incremental
        end
      end
    end

    def full
      begin
        log('begin ...')
        revoke_db_privs
        if should_restart?
          log("restarting full load...")
        else
          log("initiating full load...")
          AdminBase.connection.truncate('study_xml_records')
          @client.download_xml_files
          truncate_tables
        end
        remove_indexes  # Index significantly slow the load process.
        study_counts[:should_add]=StudyXmlRecord.count
        study_counts[:should_change]=0
        @client.populate_studies
        finalize_full_load
      rescue  Exception => e
        study_counts[:processed]=Study.count
        puts ">>>>>>>>>>> Full load failed:  #{e}"
        grant_db_privs
        load_event.complete({:status=>'failed', :problems=> e.to_s, :study_counts=> study_counts})
        send_notification
      end
    end

    def finalize_full_load
      remove_indexes  # Make sure indexes are gone before trying to add them.
      add_indexes
      grant_db_privs
      create_calculated_values
      run_sanity_checks
      refresh_data_definitions
      take_snapshot
      create_flat_files
      study_counts[:processed]=Study.count
      load_event.complete({:study_counts=>study_counts})
      send_notification
    end

    def indexes
      [<commit_msg>Add convenience method that lets us repopulate admin tables with one command.<commit_after>module ClinicalTrials
  class Updater
    attr_reader :params, :load_event, :client, :study_counts, :study_filter

    def initialize(params={})
      @params=params
      type=(params[:event_type] ? params[:event_type] : 'incremental')
      if params[:restart]
        puts("Restarting the load...")
        # don't allow filtering by nct ID unless it's a restart because filtering is done so that multiple loads can run simultaneously
        # if multiple jobs running for initial full load - they would step on each ther when study_xml_records table gets truncated
        @study_filter=@params[:study_filter]
        record_type="Restart #{@study_filter}"
      else
        puts("Starting the #{type} load...")
        record_type=type
      end
      @client = ClinicalTrials::Client.new
      @load_event = LoadEvent.create({:event_type=>record_type,:status=>'running',:description=>'',:problems=>''})
      @study_counts={:should_add=>0,:should_change=>0,:processed=>0,:count_down=>0}
      self
    end

    def run
      ActiveRecord::Base.logger=nil
      if study_filter
        @client.populate_studies(study_filter)
        load_event.complete({:new_studies=> Study.count})
      else
        case params[:event_type]
        when 'full'
          full
        when 'finalize'
          finalize_full_load
        else
          incremental
        end
      end
    end

    def full
      begin
        log('begin ...')
        revoke_db_privs
        if should_restart?
          log("restarting full load...")
        else
          log("initiating full load...")
          AdminBase.connection.truncate('study_xml_records')
          @client.download_xml_files
          truncate_tables
        end
        remove_indexes  # Index significantly slow the load process.
        study_counts[:should_add]=StudyXmlRecord.count
        study_counts[:should_change]=0
        @client.populate_studies
        finalize_full_load
      rescue  Exception => e
        study_counts[:processed]=Study.count
        puts ">>>>>>>>>>> Full load failed:  #{e}"
        grant_db_privs
        load_event.complete({:status=>'failed', :problems=> e.to_s, :study_counts=> study_counts})
        send_notification
      end
    end

    def finalize_full_load
      remove_indexes  # Make sure indexes are gone before trying to add them.
      add_indexes
      grant_db_privs
      create_calculated_values
      run_sanity_checks
      refresh_data_definitions
      take_snapshot
      create_flat_files
      study_counts[:processed]=Study.count
      load_event.complete({:study_counts=>study_counts})
      send_notification
    end

    def self.populate_admin
      SanityCheck.populate
      DataDefinition.populate
      DatabaseActivity.populate
    end

    def indexes
      [
{'fraction_non_alphanumeric': 0.09024040186580552, 'fraction_numerical': 0.001794043774668102, 'mean_word_length': 2.596774193548387, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4995411', 'n_tokens_mistral': 1657, 'n_tokens_neox': 1589, 'n_words': 363}