<commit_before>require 'spec_helper'
describe Workers::GatherOpenGraphData do
  before do
    @ogsite_title = 'Homepage'
    @ogsite_type = 'website'
    @ogsite_image = '/img/something.png'
    @ogsite_url = 'http://www.we-support-open-graph.com'
    @ogsite_description = 'Homepage'

    @ogsite_body =
      "<html><head><title>#{@ogsite_title}</title>
      <meta property=\"og:title\" content=\"#{@ogsite_title}\"/>
      <meta property=\"og:type\" content=\"#{@ogsite_type}\" />
      <meta property=\"og:image\" content=\"#{@ogsite_image}\" />
      <meta property=\"og:url\" content=\"#{@ogsite_url}\" />
      <meta property=\"og:description\" content=\"#{@ogsite_description}\" />
      </head><body></body></html>"

    @no_open_graph_url = 'http://www.we-do-not-support-open-graph.com/index.html'

    @status_message = FactoryGirl.create(:status_message)

    stub_request(:get, @ogsite_url).to_return(:status => 200, :body => @ogsite_body)
    stub_request(:get, @no_open_graph_url).to_return(:status => 200, :body => '<html><body>hello there</body></html>')
  end

  describe '.perform' do
    it 'requests not data from the internet' do
      Workers::GatherOpenGraphData.new.perform(@status_message.id, @ogsite_url)

      expect(a_request(:get, @ogsite_url)).to have_been_made
    end

    it 'requests not data from the internet only once' do
      2.times do |n|
        Workers::GatherOpenGraphData.new.perform(@status_message.id, @ogsite_url)
      end

      expect(a_request(:get, @ogsite_url)).to have_been_made.times(1)
    end

    it 'creates one cache entry' do
      Workers::GatherOpenGraphData.new.perform(@status_message.id, @ogsite_url)

      ogc = OpenGraphCache.find_by_url(@ogsite_url)

      expect(ogc.title).to eq(@ogsite_title)
      expect(ogc.ob_type).to eq(@ogsite_type)
      expect(ogc.image).to eq(@ogsite_url + @ogsite_image)
      expect(ogc.url).to eq(@ogsite_url)
      expect(ogc.description).to eq(@ogsite_description)

      Workers::GatherOpenGraphData.new.perform(@status_message.id, @ogsite_url)
      expect(OpenGraphCache.where(url: @ogsite_url).count).to eq(1)
    end

    it 'creates no cache entry for unsupported pages' do
      Workers::GatherOpenGraphData.new.perform(@status_message.id, @no_open_graph_url)

      expect(OpenGraphCache.find_by_url(@no_open_graph_url)).to be_nil
    end

    it 'gracefully handles a deleted post' do
      expect {
        Workers::GatherOpenGraphData.new.perform(0, @ogsite_url)
      }.to_not raise_error
    end
  end
end<commit_msg>Added test for truncation to gather OG data spec<commit_after>require 'spec_helper'
describe Workers::GatherOpenGraphData do
  before do
    @ogsite_title = 'Homepage'
    @ogsite_type = 'website'
    @ogsite_image = '/img/something.png'
    @ogsite_url = 'http://www.we-support-open-graph.com'
    @ogsite_description = 'Homepage'

    @ogsite_body =
      "<html><head><title>#{@ogsite_title}</title>
      <meta property=\"og:title\" content=\"#{@ogsite_title}\"/>
      <meta property=\"og:type\" content=\"#{@ogsite_type}\" />
      <meta property=\"og:image\" content=\"#{@ogsite_image}\" />
      <meta property=\"og:url\" content=\"#{@ogsite_url}\" />
      <meta property=\"og:description\" content=\"#{@ogsite_description}\" />
      </head><body></body></html>"

    @oglong_title = "D" * 256
    @oglong_url = 'http://www.we-are-too-long.com'
    @oglong_body =
      "<html><head><title>#{@oglong_title}</title>
      <meta property=\"og:title\" content=\"#{@oglong_title}\"/>
      <meta property=\"og:type\" content=\"#{@ogsite_type}\" />
      <meta property=\"og:image\" content=\"#{@ogsite_image}\" />
      <meta property=\"og:url\" content=\"#{@oglong_url}\" />
      <meta property=\"og:description\" content=\"#{@ogsite_description}\" />
      </head><body></body></html>"

    @no_open_graph_url = 'http://www.we-do-not-support-open-graph.com/index.html'

    @status_message = FactoryGirl.create(:status_message)

    stub_request(:get, @ogsite_url).to_return(:status => 200, :body => @ogsite_body)
    stub_request(:get, @no_open_graph_url).to_return(:status => 200, :body => '<html><body>hello there</body></html>')
    stub_request(:get, @oglong_url).to_return(:status => 200, :body => @oglong_body)

  end

  describe '.perform' do
    it 'requests not data from the internet' do
      Workers::GatherOpenGraphData.new.perform(@status_message.id, @ogsite_url)

      expect(a_request(:get, @ogsite_url)).to have_been_made
    end

    it 'requests not data from the internet only once' do
      2.times do |n|
        Workers::GatherOpenGraphData.new.perform(@status_message.id, @ogsite_url)
      end

      expect(a_request(:get, @ogsite_url)).to have_been_made.times(1)
    end

    it 'creates one cache entry' do
      Workers::GatherOpenGraphData.new.perform(@status_message.id, @ogsite_url)

      ogc = OpenGraphCache.find_by_url(@ogsite_url)

      expect(ogc.title).to eq(@ogsite_title)
      expect(ogc.ob_type).to eq(@ogsite_type)
      expect(ogc.image).to eq(@ogsite_url + @ogsite_image)
      expect(ogc.url).to eq(@ogsite_url)
      expect(ogc.description).to eq(@ogsite_description)

      Workers::GatherOpenGraphData.new.perform(@status_message.id, @ogsite_url)
      expect(OpenGraphCache.where(url: @ogsite_url).count).to eq(1)
    end

    it 'creates no cache entry for unsupported pages' do
      Workers::GatherOpenGraphData.new.perform(@status_message.id, @no_open_graph_url)

      expect(OpenGraphCache.find_by_url(@no_open_graph_url)).to be_nil
    end

    it 'gracefully handles a deleted post' do
      expect {
        Workers::GatherOpenGraphData.new.perform(0, @ogsite_url)
      }.to_not raise_error
    end
    it 'truncates + inserts titles that are too long' do
      Workers::GatherOpenGraphData.new.perform(@status_message.id, @oglong_url)
      ogc = OpenGraphCache.find_by_url(@oglong_url)
      expect(ogc).to be_truthy
      expect(ogc.title.length).to be <= 255
    end
  end
end
{'fraction_non_alphanumeric': 0.1557336004006009, 'fraction_numerical': 0.004840594224670339, 'mean_word_length': 3.770700636942675, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3096166', 'n_tokens_mistral': 2097, 'n_tokens_neox': 2050, 'n_words': 342}