<commit_before>require "spec_helper"

describe ActsAsElasticModel do

  before(:each) { enable_elastic_indexing([ Observation, Taxon, Identification ]) }
  after(:each) { disable_elastic_indexing([ Observation, Taxon, Identification ]) }

  describe "callbacks" do
    it "properly indexes the document on create" do
      obs = Observation.make!
      result = Observation.elastic_search( where: { id: obs.id } )
      expect( result.count ).to eq 1
      expect( result.first.class ).to eq Elasticsearch::Model::Response::Result
      expect( result.first.id.to_i ).to eq obs.id
      expect( Time.parse(result.first.created_at).round ).to eq obs.created_at.round
      expect( result.first.user.login ).to eq obs.user.login
    end

    it "properly deletes the document on destroy" do
      obs = Observation.make!
      expect( Observation.elastic_search( where: { id: obs.id } ).count ).to eq 1
      obs.destroy
      expect( Observation.elastic_search( where: { id: obs.id } ).count ).to eq 0
    end

    it "properly deletes the document on commit" do
      obs = Observation.make!
      expect(obs).to receive(:elastic_delete!).twice
      # we need to destroy to make sure we hit the after_commit on: destroy
      obs.destroy!
      # forcing the commit, which doesn't usually happen in specs
      obs.run_callbacks(:commit)
    end
  end

  describe "class methods" do
    describe "elastic_search" do
      it "searches for bool: { } as a wildcard query" do
        expect(Observation.__elasticsearch__).to receive(:search).with(
          { query: { constant_score: { query: { bool: { } } } } }).and_return(true)
        Observation.elastic_search( )
      end

      it "adds matches to the query" do
        expect(Observation.__elasticsearch__).to receive(:search).with(
          { query: { constant_score: { query: { bool: {
            must: [ { match: { id: 5 } } ] } } } } }).and_return(true)
        Observation.elastic_search(where: { id: 5 })
      end

      it "adds terms matches to the query" do
        expect(Observation.__elasticsearch__).to receive(:search).with(
          { query: { constant_score: { query: { bool: {
            must: [ { terms: { id: [ 1, 3 ] } } ] } } } } }).and_return(true)
        Observation.elastic_search(where: { id: [ 1, 3] })
      end

      it "adds envelope filters" do
        expect(Observation.__elasticsearch__).to receive(:search).with(
          { query: { constant_score: { query: { bool: { must: [
            { geo_shape: { geojson: { shape: {
              type: "envelope", coordinates: [[-180, -90], [180, 88]]}}}}]}}}}}).and_return(true)
        Observation.elastic_search(filters: [ { envelope: { geojson: { nelat: 88 }}}])
      end

      it "adds sorts to the query" do
        expect(Observation.__elasticsearch__).to receive(:search).with(
          { query: { constant_score: { query: { bool: { } } } },
            sort: { score: :desc } }).and_return(true)
        Observation.elastic_search(sort: { score: :desc })
      end

      it "allows certain fields to be specified" do
        expect(Observation.__elasticsearch__).to receive(:search).with(
          { query: { constant_score: { query: { bool: { } } } },
            _source: [ "id", "description" ] }).and_return(true)
        Observation.elastic_search(source: [ "id", "description" ])
      end

      it "adds aggregations to the query" do
        expect(Observation.__elasticsearch__).to receive(:search).with(
          { query: { constant_score: { query: { bool: { } } } },
            aggs: { colors: { terms: {
              field: :"colors.id", size: 10 } } } }).and_return(true)
        Observation.elastic_search(aggregate: { colors: { "colors.id": 10 } } )
      end
    end

    describe "elastic_paginate" do
      it "returns a WillPaginate collection" do
        expect(Observation.elastic_paginate).to be_a WillPaginate::Collection
      end
      it "does not modify the options it receives" do
        options = { max_id: 1 }
        Observation.elastic_paginate( options )
        expect( options[:where] ).to be_nil
      end
    end

    describe "elastic_index!" do
      it "indexes instances of a class" do
        obs = Observation.make!
        expect( Observation.elastic_search( where: { id: obs.id } ).count ).to eq 1
        obs.elastic_delete!
        expect( Observation.elastic_search( where: { id: obs.id } ).count ).to eq 0
        Observation.elastic_index!
        expect( Observation.elastic_search( where: { id: obs.id } ).count ).to eq 1
      end

      it "accepts a scope" do
        obs1 = Observation.make!
        obs2 = Observation.make!
        obs1.elastic_delete!
        obs2.elastic_delete!
        expect( Observation.elastic_search( ).count ).to eq 0
        Observation.elastic_index!(scope: Observation.where(id: obs2))
        expect( Observation.elastic_search( ).count ).to eq 1
        expect( Observation.elastic_search( ).first.id.to_i ).to eq obs2.id
      end

      it "accepts an array of ids" do
        obs1 = Observation.make!
        obs2 = Observation.make!
        obs1.elastic_delete!
        obs2.elastic_delete!
        expect( Observation.elastic_search( ).count ).to eq 0
        Observation.elastic_index!(ids: [ obs2.id ])
        expect( Observation.elastic_search( ).count ).to eq 1
        expect( Observation.elastic_search( ).first.id.to_i ).to eq obs2.id
      end

      it "calls prepare_batch_for_index if it exists" do
        # Taxon uses prepare_batch_for_index, so it good for this test
        taxon = Taxon.make!
        expect(Taxon).to receive(:prepare_batch_for_index)
        Taxon.elastic_index!
      end

      it "exceptions are caught silently" do
        expect(Observation.__elasticsearch__.client).to receive(:bulk).
          and_raise(Elasticsearch::Transport::Transport::Errors::BadRequest)
        obs = Observation.make!
        obs.elastic_delete!
        Observation.elastic_index!
        expect( Observation.elastic_search( ).count ).to eq 0
      end

      it "sets last_index_at for Observations" do
        obs = Observation.make!
        obs.update_column(:last_indexed_at, 1.year.ago)
        expect( obs.last_indexed_at ).to be < 1.minute.ago
        Observation.elastic_index!
        obs.reload
        expect( obs.last_indexed_at ).to be > 1.minute.ago
      end

      it "can be delayed" do
        obs = Observation.make!
        obs.update_column(:last_indexed_at, 1.year.ago)
        expect( obs.last_indexed_at ).to be < 1.minute.ago
        Observation.elastic_index!( delay: true )
        expect( obs.last_indexed_at ).to be < 1.minute.ago
        Delayed::Worker.new.work_off
        obs.reload
        expect( obs.last_indexed_at ).to be > 1.minute.ago
      end

      it "doesn't re-index obs indexed more than 5 minutes after delayed index request" do
        obs = Observation.make!
        Observation.elastic_index!( delay: true )
        less_than_five_minutes = 4.minutes.from_now
        obs.update_column(:last_indexed_at, less_than_five_minutes)
        Delayed::Worker.new.work_off
        obs.reload
        # obs indexed less than 5 minutes after delay request are re-indexed
        expect( obs.last_indexed_at ).to_not eq less_than_five_minutes

        Observation.elastic_index!( delay: true )
        more_than_five_minutes = 10.minutes.from_now
        obs.update_column(:last_indexed_at, more_than_five_minutes)
        Delayed::Worker.new.work_off
        obs.reload
        # obs index more than 5 minutes after delay request aren't re-indexed
        expect( obs.last_indexed_at ).to eq more_than_five_minutes
      end

    end<commit_msg>account for Travis precision variance<commit_after>require "spec_helper"

describe ActsAsElasticModel do

  before(:each) { enable_elastic_indexing([ Observation, Taxon, Identification ]) }
  after(:each) { disable_elastic_indexing([ Observation, Taxon, Identification ]) }

  describe "callbacks" do
    it "properly indexes the document on create" do
      obs = Observation.make!
      result = Observation.elastic_search( where: { id: obs.id } )
      expect( result.count ).to eq 1
      expect( result.first.class ).to eq Elasticsearch::Model::Response::Result
      expect( result.first.id.to_i ).to eq obs.id
      expect( Time.parse(result.first.created_at).round ).to eq obs.created_at.round
      expect( result.first.user.login ).to eq obs.user.login
    end

    it "properly deletes the document on destroy" do
      obs = Observation.make!
      expect( Observation.elastic_search( where: { id: obs.id } ).count ).to eq 1
      obs.destroy
      expect( Observation.elastic_search( where: { id: obs.id } ).count ).to eq 0
    end

    it "properly deletes the document on commit" do
      obs = Observation.make!
      expect(obs).to receive(:elastic_delete!).twice
      # we need to destroy to make sure we hit the after_commit on: destroy
      obs.destroy!
      # forcing the commit, which doesn't usually happen in specs
      obs.run_callbacks(:commit)
    end
  end

  describe "class methods" do
    describe "elastic_search" do
      it "searches for bool: { } as a wildcard query" do
        expect(Observation.__elasticsearch__).to receive(:search).with(
          { query: { constant_score: { query: { bool: { } } } } }).and_return(true)
        Observation.elastic_search( )
      end

      it "adds matches to the query" do
        expect(Observation.__elasticsearch__).to receive(:search).with(
          { query: { constant_score: { query: { bool: {
            must: [ { match: { id: 5 } } ] } } } } }).and_return(true)
        Observation.elastic_search(where: { id: 5 })
      end

      it "adds terms matches to the query" do
        expect(Observation.__elasticsearch__).to receive(:search).with(
          { query: { constant_score: { query: { bool: {
            must: [ { terms: { id: [ 1, 3 ] } } ] } } } } }).and_return(true)
        Observation.elastic_search(where: { id: [ 1, 3] })
      end

      it "adds envelope filters" do
        expect(Observation.__elasticsearch__).to receive(:search).with(
          { query: { constant_score: { query: { bool: { must: [
            { geo_shape: { geojson: { shape: {
              type: "envelope", coordinates: [[-180, -90], [180, 88]]}}}}]}}}}}).and_return(true)
        Observation.elastic_search(filters: [ { envelope: { geojson: { nelat: 88 }}}])
      end

      it "adds sorts to the query" do
        expect(Observation.__elasticsearch__).to receive(:search).with(
          { query: { constant_score: { query: { bool: { } } } },
            sort: { score: :desc } }).and_return(true)
        Observation.elastic_search(sort: { score: :desc })
      end

      it "allows certain fields to be specified" do
        expect(Observation.__elasticsearch__).to receive(:search).with(
          { query: { constant_score: { query: { bool: { } } } },
            _source: [ "id", "description" ] }).and_return(true)
        Observation.elastic_search(source: [ "id", "description" ])
      end

      it "adds aggregations to the query" do
        expect(Observation.__elasticsearch__).to receive(:search).with(
          { query: { constant_score: { query: { bool: { } } } },
            aggs: { colors: { terms: {
              field: :"colors.id", size: 10 } } } }).and_return(true)
        Observation.elastic_search(aggregate: { colors: { "colors.id": 10 } } )
      end
    end

    describe "elastic_paginate" do
      it "returns a WillPaginate collection" do
        expect(Observation.elastic_paginate).to be_a WillPaginate::Collection
      end
      it "does not modify the options it receives" do
        options = { max_id: 1 }
        Observation.elastic_paginate( options )
        expect( options[:where] ).to be_nil
      end
    end

    describe "elastic_index!" do
      it "indexes instances of a class" do
        obs = Observation.make!
        expect( Observation.elastic_search( where: { id: obs.id } ).count ).to eq 1
        obs.elastic_delete!
        expect( Observation.elastic_search( where: { id: obs.id } ).count ).to eq 0
        Observation.elastic_index!
        expect( Observation.elastic_search( where: { id: obs.id } ).count ).to eq 1
      end

      it "accepts a scope" do
        obs1 = Observation.make!
        obs2 = Observation.make!
        obs1.elastic_delete!
        obs2.elastic_delete!
        expect( Observation.elastic_search( ).count ).to eq 0
        Observation.elastic_index!(scope: Observation.where(id: obs2))
        expect( Observation.elastic_search( ).count ).to eq 1
        expect( Observation.elastic_search( ).first.id.to_i ).to eq obs2.id
      end

      it "accepts an array of ids" do
        obs1 = Observation.make!
        obs2 = Observation.make!
        obs1.elastic_delete!
        obs2.elastic_delete!
        expect( Observation.elastic_search( ).count ).to eq 0
        Observation.elastic_index!(ids: [ obs2.id ])
        expect( Observation.elastic_search( ).count ).to eq 1
        expect( Observation.elastic_search( ).first.id.to_i ).to eq obs2.id
      end

      it "calls prepare_batch_for_index if it exists" do
        # Taxon uses prepare_batch_for_index, so it good for this test
        taxon = Taxon.make!
        expect(Taxon).to receive(:prepare_batch_for_index)
        Taxon.elastic_index!
      end

      it "exceptions are caught silently" do
        expect(Observation.__elasticsearch__.client).to receive(:bulk).
          and_raise(Elasticsearch::Transport::Transport::Errors::BadRequest)
        obs = Observation.make!
        obs.elastic_delete!
        Observation.elastic_index!
        expect( Observation.elastic_search( ).count ).to eq 0
      end

      it "sets last_index_at for Observations" do
        obs = Observation.make!
        obs.update_column(:last_indexed_at, 1.year.ago)
        expect( obs.last_indexed_at ).to be < 1.minute.ago
        Observation.elastic_index!
        obs.reload
        expect( obs.last_indexed_at ).to be > 1.minute.ago
      end

      it "can be delayed" do
        obs = Observation.make!
        obs.update_column(:last_indexed_at, 1.year.ago)
        expect( obs.last_indexed_at ).to be < 1.minute.ago
        Observation.elastic_index!( delay: true )
        expect( obs.last_indexed_at ).to be < 1.minute.ago
        Delayed::Worker.new.work_off
        obs.reload
        expect( obs.last_indexed_at ).to be > 1.minute.ago
      end

      it "doesn't re-index obs indexed more than 5 minutes after delayed index request" do
        obs = Observation.make!
        Observation.elastic_index!( delay: true )
        less_than_five_minutes = 4.minutes.from_now
        obs.update_column(:last_indexed_at, less_than_five_minutes)
        Delayed::Worker.new.work_off
        obs.reload
        # obs indexed less than 5 minutes after delay request are re-indexed
        expect( obs.last_indexed_at ).to_not eq less_than_five_minutes

        Observation.elastic_index!( delay: true )
        more_than_five_minutes = 10.minutes.from_now
        obs.update_column(:last_indexed_at, more_than_five_minutes)
        Delayed::Worker.new.work_off
        obs.reload
        # obs index more than 5 minutes after delay request aren't re-indexed
        expect( ( obs.last_indexed_at - more_than_five_minutes ).abs ).to be < 1
      end

    end
{'fraction_non_alphanumeric': 0.10952567109721137, 'fraction_numerical': 0.007753453218660411, 'mean_word_length': 2.543167128347184, 'pattern_counts': {'":': 2, '<': 10, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7584760', 'n_tokens_mistral': 4821, 'n_tokens_neox': 4560, 'n_words': 1209}