<commit_before>require 'pry'
require 'minitest/autorun'
require_relative '../lib/ncbo_resource_index'

# Kept for test data consistency
Annotator = RI::Population

# Less logging
$ri_log_level = Logger::FATAL

require_relative 'shared_data'
require_relative 'population_setup' unless $skip_population_setup
require_relative 'simulate_failures'

# Prevent Persisted::Hash from writing data while testing
Persisted::Hash.prevent_persist unless $skip_prevent_persist

module ResourceIndex
  def self.setup_sql_client
    if RUBY_PLATFORM == "java"
      @client = Sequel.connect("jdbc:sqlite:ri_test.db")
    else
      @client = Sequel.connect("sqlite:ri_test.db")
    end
  end
end

module RI
  class TestCase < Minitest::Test
    TOTAL_ES_RECORDS = 928

    def setup
      Dir.glob(Dir.pwd + "/ae_test*resume").each {|f| File.delete(f)}
      RI::Population::Document.fail_on_index(false)
      RI.config(sqlite: true, resource_store: "test_resource_store")
      RI.db.create_table :obr_resource do
        primary_key :id
        String :name
        String :resource_id
        String :structure
        String :main_context
        String :url
        String :element_url
        String :description
        String :logo
        Integer :dictionary_id
        Integer :total_element
        Time :last_update_date
        Time :workflow_completed_date
      end
      RI.db.run(RESOURCES_TEST_DATA)
      RI.db.create_table :obr_ae_test_element do
        primary_key :id
        String :local_element_id
        Integer :dictionary_id
        String :ae_name
        String :ae_description
        String :ae_species
        String :ae_experiment_type
      end
      RI.db.run(DOCUMENTS_TEST_DATA.force_encoding('UTF-8'))
      RI.db.create_table :obs_ontology do
        primary_key :id
        String :local_ontology_id
        String :name
        String :version
        String :description
        Integer :status
        String :virtual_ontology_id
        String :format
        Integer :dictionary_id
      end
      RI.db.run(ONTOLOGIES_TABLE)
      RI.db.create_table :obs_concept do
        primary_key :id
        String :local_concept_id
        Integer :ontology_id
        Integer :is_toplevel
        String :full_id
      end
      RI.db.run(CONCEPTS_TABLE)
    end

    def teardown
      RI::Population::Document.fail_on_index(false)
      RI.db[:obr_resource].delete
      RI.db[:obr_ae_test_element].delete
      RI.db[:obs_ontology].delete
      RI.db[:obs_concept].delete
      if @es && @index_id
        @es.indices.delete index: @index_id
      end
      db_file = Dir.pwd+"/ri_test.db"
      File.delete(db_file) if File.exist?(db_file)
    end
  end
end
<commit_msg>Delete the index used to store resource data<commit_after>require 'pry'
require 'minitest/autorun'
require_relative '../lib/ncbo_resource_index'

# Kept for test data consistency
Annotator = RI::Population

# Less logging
$ri_log_level = Logger::FATAL

require_relative 'shared_data'
require_relative 'population_setup' unless $skip_population_setup
require_relative 'simulate_failures'

# Prevent Persisted::Hash from writing data while testing
Persisted::Hash.prevent_persist unless $skip_prevent_persist

module ResourceIndex
  def self.setup_sql_client
    if RUBY_PLATFORM == "java"
      @client = Sequel.connect("jdbc:sqlite:ri_test.db")
    else
      @client = Sequel.connect("sqlite:ri_test.db")
    end
  end
end

module RI
  class TestCase < Minitest::Test
    TOTAL_ES_RECORDS = 928

    def setup
      Dir.glob(Dir.pwd + "/ae_test*resume").each {|f| File.delete(f)}
      RI::Population::Document.fail_on_index(false)
      @resource_store = "test_resource_store_#{Time.now.to_i}"
      RI.config(sqlite: true, resource_store: @resource_store)
      RI.db.create_table :obr_resource do
        primary_key :id
        String :name
        String :resource_id
        String :structure
        String :main_context
        String :url
        String :element_url
        String :description
        String :logo
        Integer :dictionary_id
        Integer :total_element
        Time :last_update_date
        Time :workflow_completed_date
      end
      RI.db.run(RESOURCES_TEST_DATA)
      RI.db.create_table :obr_ae_test_element do
        primary_key :id
        String :local_element_id
        Integer :dictionary_id
        String :ae_name
        String :ae_description
        String :ae_species
        String :ae_experiment_type
      end
      RI.db.run(DOCUMENTS_TEST_DATA.force_encoding('UTF-8'))
      RI.db.create_table :obs_ontology do
        primary_key :id
        String :local_ontology_id
        String :name
        String :version
        String :description
        Integer :status
        String :virtual_ontology_id
        String :format
        Integer :dictionary_id
      end
      RI.db.run(ONTOLOGIES_TABLE)
      RI.db.create_table :obs_concept do
        primary_key :id
        String :local_concept_id
        Integer :ontology_id
        Integer :is_toplevel
        String :full_id
      end
      RI.db.run(CONCEPTS_TABLE)
    end

    def teardown
      RI::Population::Document.fail_on_index(false)
      RI.db[:obr_resource].delete
      RI.db[:obr_ae_test_element].delete
      RI.db[:obs_ontology].delete
      RI.db[:obs_concept].delete
      if @es && @index_id
        @es.indices.delete index: @index_id
        @es.indices.delete index: @resource_store
      end
      db_file = Dir.pwd+"/ri_test.db"
      File.delete(db_file) if File.exist?(db_file)
    end
  end
end

{'fraction_non_alphanumeric': 0.07996389891696751, 'fraction_numerical': 0.001444043321299639, 'mean_word_length': 2.723790322580645, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1410516', 'n_tokens_mistral': 1825, 'n_tokens_neox': 1745, 'n_words': 370}