<commit_before>require 'open3'
module Util
  class DbManager

    attr_accessor :con, :stage_con, :pub_con, :event

    def initialize(params={})
      # Should only manage content of ctgov db schema
      if params[:event]
        @event = params[:event]
      else
        @event = Support::LoadEvent.create({:event_type=>'',:status=>'',:description=>'',:problems=>''})
      end
    end

    def dump_database
      fm=Util::FileManager.new
      # First populate db named 'aact' from background db so the dump file will be configured to restore db named aact
      psql_file="#{fm.dump_directory}/aact.psql"
      File.delete(psql_file) if File.exist?(psql_file)
      # pg_dump that works on postgres 10.3
      #cmd="pg_dump --no-owner --no-acl --host=localhost --username=#{ENV['AACT_DB_SUPER_USERNAME']} --dbname=aact_back --schema=ctgov > #{psql_file}"
      # pg_dump that works on postgres 9.2.23 - which is what's running on servers as of 4/20/18
      cmd="pg_dump --no-owner --no-acl --host=localhost --username #{ENV['AACT_DB_SUPER_USERNAME']} --schema=ctgov  #{ENV['AACT_BACK_DATABASE_URL']} > #{psql_file}"
      run_command_line(cmd)

      # clear out previous ctgov content from staging db
      log "recreating ctgov schema in aact staging database..."
      terminate_stage_db_sessions
      begin
        stage_con.execute('DROP SCHEMA ctgov CASCADE;')
      rescue
      end

      # refresh staging db
      log "refreshing aact staging database..."
      cmd="psql -h localhost  #{ENV['AACT_STAGE_DATABASE_URL']} < #{psql_file} > /dev/null"
      run_command_line(cmd)

      File.delete(fm.pg_dump_file) if File.exist?(fm.pg_dump_file)
      cmd="pg_dump #{ENV['AACT_STAGE_DATABASE_URL']} -v -h localhost -p 5432 -U #{ENV['AACT_DB_SUPER_USERNAME']} --no-password --clean --exclude-table ar_internal_metadata --exclude-table schema_migrations --schema ctgov --schema proj --schema proj_tag_nephrology --schema proj_anderson -b -c -C -Fc -f #{fm.pg_dump_file}"
      run_command_line(cmd)

      ActiveRecord::Base.establish_connection(ENV["AACT_BACK_DATABASE_URL"]).connection
    end

    def refresh_public_db
      begin
        success_code=true
        revoke_db_privs
        terminate_db_sessions
        dump_file_name=Util::FileManager.new.pg_dump_file
        return nil if dump_file_name.nil?
        cmd="pg_restore -c -j 5 -v -h #{public_host_name} -p 5432 -U #{ENV['AACT_DB_SUPER_USERNAME']}  -d #{public_db_name} #{dump_file_name}"
        run_command_line(cmd)

        terminate_alt_db_sessions
        cmd="pg_restore -c -j 5 -v -h #{public_host_name} -p 5432 -U #{ENV['AACT_DB_SUPER_USERNAME']}  -d aact_alt #{dump_file_name}"
        run_command_line(cmd)

        grant_db_privs
        return success_code
      rescue => error
        msg="#{error.message} (#{error.class} #{error.backtrace}"<commit_msg>aact-808:  Collaborate with AACT Projects to drop all cross-schema views before db restore & recreate them when it’s done.  AACT Projects is expected to have a function named create_views that must create all cross-schema views.  Core AACT will drop all views in schemas that have a proj_ prefix, except those named data_definitions (because these are standard project views that don’t cross schemas.)<commit_after>require 'open3'
module Util
  class DbManager

    attr_accessor :con, :stage_con, :pub_con, :event

    def initialize(params={})
      # Should only manage content of ctgov db schema
      if params[:event]
        @event = params[:event]
      else
        @event = Support::LoadEvent.create({:event_type=>'',:status=>'',:description=>'',:problems=>''})
      end
    end

    def drop_project_views
      con=ActiveRecord::Base.establish_connection(ENV["AACT_PUBLIC_DATABASE_URL"]).connection
      # Create a collection of 'DROP VIEW commands for each project view.
      creation_cmd="
        SELECT 'DROP VIEW ' || table_schema || '.' || table_name || ';'
          FROM information_schema.views
         WHERE table_schema LIKE 'proj%'
           AND table_name != 'data_definitions';"
      cmds=con.execute(creation_cmd)
      cmds.each{ |cmd| con.execute(cmd['?column?']) }
      con.disconnect!
    end

    def create_project_views
      con=ActiveRecord::Base.establish_connection(ENV["AACT_PUBLIC_DATABASE_URL"]).connection
      cmd="select create_views();"
      con.execute(cmd)
      con.disconnect!
    end

    def dump_database
      fm=Util::FileManager.new
      # First populate db named 'aact' from background db so the dump file will be configured to restore db named aact
      psql_file="#{fm.dump_directory}/aact.psql"
      File.delete(psql_file) if File.exist?(psql_file)
      # pg_dump that works on postgres 10.3
      #cmd="pg_dump --no-owner --no-acl --host=localhost --username=#{ENV['AACT_DB_SUPER_USERNAME']} --dbname=aact_back --schema=ctgov > #{psql_file}"
      # pg_dump that works on postgres 9.2.23 - which is what's running on servers as of 4/20/18
      cmd="pg_dump --no-owner --no-acl --host=localhost --username #{ENV['AACT_DB_SUPER_USERNAME']} --schema=ctgov  #{ENV['AACT_BACK_DATABASE_URL']} > #{psql_file}"
      run_command_line(cmd)

      # clear out previous ctgov content from staging db
      log "recreating ctgov schema in aact staging database..."
      terminate_stage_db_sessions
      begin
        stage_con.execute('DROP SCHEMA ctgov CASCADE;')
      rescue
      end

      # refresh staging db
      log "refreshing aact staging database..."
      cmd="psql -h localhost  #{ENV['AACT_STAGE_DATABASE_URL']} < #{psql_file} > /dev/null"
      run_command_line(cmd)

      File.delete(fm.pg_dump_file) if File.exist?(fm.pg_dump_file)
      cmd="pg_dump #{ENV['AACT_STAGE_DATABASE_URL']} -v -h localhost -p 5432 -U #{ENV['AACT_DB_SUPER_USERNAME']} --no-password --clean --exclude-table ar_internal_metadata --exclude-table schema_migrations --schema ctgov --schema proj --schema proj_tag_nephrology --schema proj_anderson -b -c -C -Fc -f #{fm.pg_dump_file}"
      run_command_line(cmd)

      ActiveRecord::Base.establish_connection(ENV["AACT_BACK_DATABASE_URL"]).connection
    end

    def refresh_public_db
      begin
        success_code=true
        revoke_db_privs
        terminate_db_sessions
        drop_project_views
        dump_file_name=Util::FileManager.new.pg_dump_file
        return nil if dump_file_name.nil?
        cmd="pg_restore -c -j 5 -v -h #{public_host_name} -p 5432 -U #{ENV['AACT_DB_SUPER_USERNAME']}  -d #{public_db_name} #{dump_file_name}"
        run_command_line(cmd)

        terminate_alt_db_sessions
        cmd="pg_restore -c -j 5 -v -h #{public_host_name} -p 5432 -U #{ENV['AACT_DB_SUPER_USERNAME']}  -d aact_alt #{dump_file_name}"
        run_command_line(cmd)
        create_project_views

        grant_db_privs
        return success_code
      rescue => error
        msg="#{error.message} (#{error.class} #{error.backtrace}"
{'fraction_non_alphanumeric': 0.11458786035057221, 'fraction_numerical': 0.008257279443720122, 'mean_word_length': 3.238182934315531, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3686051', 'n_tokens_mistral': 2370, 'n_tokens_neox': 2268, 'n_words': 544}