<commit_before>require "spec_helper"
# require "byebug"

describe Validation, type: :model do

  describe '#expiry_fields' do
    it "should assign a TTL field to any validation formed from an uploaded CSV file" do
      @file = mock_upload('csvs/valid.csv')
      validation = Validation.create_validation(@file)
      validation.expirable_created_at.should_not == nil
    end


    it "should assign the TTL field as a Mongoid index" do
      @file = mock_upload('csvs/valid.csv')
      validation = Validation.create_validation(@file)
      # retrieve the mongo collection associated with above created file, ensure that the expirable field is present
      validation.collection.indexes[expirable_created_at: 1].present?.should == true
    end

    it "should have an expiry value of 24 hours" do
      @file = mock_upload('csvs/valid.csv')
      validation = Validation.create_validation(@file)
      # retrieve the mongo collection associated with above created file, ensure that the expirable field is set to 24 hours
      validation.collection.indexes[expirable_created_at: 1].select{|k,v| k=="expireAfterSeconds"}.has_value?(24.hours.to_i)
    end

  end

  it "should not assign a TTL field to any validation formed from a hyperlinked CSV file" do
    mock_file("http://example.com/test.csv", 'csvs/valid.csv')
    validation = Validation.create_validation("http://example.com/test.csv")
    validation.expirable_created_at.should == nil
  end

  it "should recheck validations after two hours" do
    mock_file("http://example.com/test.csv", 'csvs/valid.csv')
    validation = Validation.create_validation("http://example.com/test.csv")
    validation.state.should == "valid"
    mock_file("http://example.com/test.csv", 'csvs/errors.csv')
    Timecop.freeze(DateTime.now + 2.hours) do
      validation.check_validation
      validation.state.should == "invalid"
    end
  end

  it "should not requeue a validation for images if revalidate is set to false" do
    mock_file("http://example.com/test.csv", 'csvs/valid.csv')
    validation = Validation.create_validation("http://example.com/test.csv")
    Validation.any_instance.should_not_receive(:delay)
    Validation.fetch_validation(validation.id, "png", false)
  end

  it "should not revalidate for html if revalidate is set to false" do
    mock_file("http://example.com/test.csv", 'csvs/valid.csv')
    validation = Validation.create_validation("http://example.com/test.csv")
    Validation.any_instance.should_not_receive(:check_validation)
    Validation.fetch_validation(validation.id, "html", false)
  end

  it "should only create one validation per url" do
    mock_file("http://example.com/test.csv", 'csvs/valid.csv')
    validation = Validation.create_validation("http://example.com/test.csv")
    Validation.count.should == 1

    validation = Validation.create_validation("http://example.com/test.csv")
    Validation.count.should == 1
  end

end<commit_msg>Simple spec test for parse options (dialect) presence<commit_after>require "spec_helper"
# require "byebug"

describe Validation, type: :model do

  describe '#expiry_fields' do
    it "should assign a TTL field to any validation formed from an uploaded CSV file" do
      @file = mock_upload('csvs/valid.csv')
      validation = Validation.create_validation(@file)
      validation.expirable_created_at.should_not == nil
    end


    it "should assign the TTL field as a Mongoid index" do
      @file = mock_upload('csvs/valid.csv')
      validation = Validation.create_validation(@file)
      # retrieve the mongo collection associated with above created file, ensure that the expirable field is present
      validation.collection.indexes[expirable_created_at: 1].present?.should == true
    end

    it "should have an expiry value of 24 hours" do
      @file = mock_upload('csvs/valid.csv')
      validation = Validation.create_validation(@file)
      # retrieve the mongo collection associated with above created file, ensure that the expirable field is set to 24 hours
      validation.collection.indexes[expirable_created_at: 1].select{|k,v| k=="expireAfterSeconds"}.has_value?(24.hours.to_i)
    end

  end

  it "should not assign a TTL field to any validation formed from a hyperlinked CSV file" do
    mock_file("http://example.com/test.csv", 'csvs/valid.csv')
    validation = Validation.create_validation("http://example.com/test.csv")
    validation.expirable_created_at.should == nil
  end

  it "should recheck validations after two hours" do
    mock_file("http://example.com/test.csv", 'csvs/valid.csv')
    validation = Validation.create_validation("http://example.com/test.csv")
    validation.state.should == "valid"
    mock_file("http://example.com/test.csv", 'csvs/errors.csv')
    Timecop.freeze(DateTime.now + 2.hours) do
      validation.check_validation
      validation.state.should == "invalid"
    end
  end

  it "should not requeue a validation for images if revalidate is set to false" do
    mock_file("http://example.com/test.csv", 'csvs/valid.csv')
    validation = Validation.create_validation("http://example.com/test.csv")
    Validation.any_instance.should_not_receive(:delay)
    Validation.fetch_validation(validation.id, "png", false)
  end

  it "should not revalidate for html if revalidate is set to false" do
    mock_file("http://example.com/test.csv", 'csvs/valid.csv')
    validation = Validation.create_validation("http://example.com/test.csv")
    Validation.any_instance.should_not_receive(:check_validation)
    Validation.fetch_validation(validation.id, "html", false)
  end

  it "should only create one validation per url" do
    mock_file("http://example.com/test.csv", 'csvs/valid.csv')
    validation = Validation.create_validation("http://example.com/test.csv")
    Validation.count.should == 1

    validation = Validation.create_validation("http://example.com/test.csv")
    Validation.count.should == 1
  end

  it "parse options should be created with validation" do
    @file = mock_upload('csvs/valid.csv')
    validation = Validation.create_validation(@file)
    validation.parse_options.should_not == nil
  end

end
{'fraction_non_alphanumeric': 0.11016810837277624, 'fraction_numerical': 0.003590664272890485, 'mean_word_length': 3.9619433198380567, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1857626', 'n_tokens_mistral': 1851, 'n_tokens_neox': 1757, 'n_words': 468}