<commit_before>require 'spec_helper'

describe Csvlint::Validator do
    
  before do
    stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :body => "")
  end
  
  context "csv dialect" do
    it "should provide sensible defaults for CSV parsing" do
      validator = Csvlint::Validator.new("http://example.com/example.csv")
      opts = validator.instance_variable_get("@csv_options")
      opts.should include({
        :col_sep => ",",
        :row_sep => :auto,
        :quote_char => '"',
        :skip_blanks => false
      })
    end
    
    it "should map CSV DDF to correct values" do
      validator = Csvlint::Validator.new("http://example.com/example.csv")
      opts = validator.dialect_to_csv_options( {
        "lineTerminator" => "\n",
        "delimiter" => "\t",
        "quoteChar" => "'"
      })
      opts.should include({
        :col_sep => "\t",
        :row_sep => "\n",
        :quote_char => "'",
        :skip_blanks => false
      })       
    end
    
  end
  
  context "when detecting headers" do
    it "should default to expecting a header" do
      validator = Csvlint::Validator.new("http://example.com/example.csv")
      expect( validator.header? ).to eql(true)
    end
    
    it "should look in CSV options to detect header" do
      opts = {
        "header" => true
      }
      validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
      expect( validator.header? ).to eql(true)
      opts = {
        "header" => false
      }
      validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
      expect( validator.header? ).to eql(false)      
    end
    
    it "should look in content-type for header" do
      stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=absent"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
      validator = Csvlint::Validator.new("http://example.com/example.csv")
      expect( validator.header? ).to eql(false)

      stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=present"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
      validator = Csvlint::Validator.new("http://example.com/example.csv")
      expect( validator.header? ).to eql(true)              
    end  
    
  end
  
  context "when validating headers" do
    it "should warn if column names aren't unique" do      
      data = StringIO.new( "minimum, minimum" )
      validator = Csvlint::Validator.new(data)
      expect( validator.validate_header(["minimum", "minimum"]) ).to eql(true)
      expect( validator.warnings.size ).to eql(1)
      expect( validator.warnings.first.type).to eql(:duplicate_column_name)
      expect( validator.warnings.first.category).to eql(:schema)
    end

    it "should warn if column names are blank" do
      data = StringIO.new( "minimum," )
      validator = Csvlint::Validator.new(data)
      
      expect( validator.validate_header(["minimum", ""]) ).to eql(true)
      expect( validator.warnings.size ).to eql(1)
      expect( validator.warnings.first.type).to eql(:empty_column_name)
      expect( validator.warnings.first.category).to eql(:schema)
    end
    
    it "should include info message about missing header when we have assumed a header" do
      data = StringIO.new( "1,2,3\r\n" )
      validator = Csvlint::Validator.new(data)
      
      expect( validator.valid? ).to eql(true)
      expect( validator.info_messages.size ).to eql(1)
      expect( validator.info_messages.first.type).to eql(:assumed_header)<commit_msg>Add tests for current undeclared_header behaviour<commit_after>require 'spec_helper'

describe Csvlint::Validator do
    
  before do
    stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :body => "")
  end
  
  context "csv dialect" do
    it "should provide sensible defaults for CSV parsing" do
      validator = Csvlint::Validator.new("http://example.com/example.csv")
      opts = validator.instance_variable_get("@csv_options")
      opts.should include({
        :col_sep => ",",
        :row_sep => :auto,
        :quote_char => '"',
        :skip_blanks => false
      })
    end
    
    it "should map CSV DDF to correct values" do
      validator = Csvlint::Validator.new("http://example.com/example.csv")
      opts = validator.dialect_to_csv_options( {
        "lineTerminator" => "\n",
        "delimiter" => "\t",
        "quoteChar" => "'"
      })
      opts.should include({
        :col_sep => "\t",
        :row_sep => "\n",
        :quote_char => "'",
        :skip_blanks => false
      })       
    end
    
  end
  
  context "when detecting headers" do
    it "should default to expecting a header" do
      validator = Csvlint::Validator.new("http://example.com/example.csv")
      expect( validator.header? ).to eql(true)
    end
    
    it "should look in CSV options to detect header" do
      opts = {
        "header" => true
      }
      validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
      expect( validator.header? ).to eql(true)
      opts = {
        "header" => false
      }
      validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
      expect( validator.header? ).to eql(false)      
    end

    it "should look in content-type for header=absent" do
      stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=absent"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
      validator = Csvlint::Validator.new("http://example.com/example.csv")
      expect( validator.header? ).to eql(false)
      expect( validator.errors.size ).to eql(0)
    end

    it "should look in content-type for header=present" do
      stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=present"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
      validator = Csvlint::Validator.new("http://example.com/example.csv")
      expect( validator.header? ).to eql(true)
      expect( validator.errors.size ).to eql(0)
    end

    it "assume header present if not specified in content type" do
      stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
      validator = Csvlint::Validator.new("http://example.com/example.csv")
      expect( validator.header? ).to eql(true)
      expect( validator.errors.size ).to eql(1)
      expect( validator.errors.first.type).to eql(:undeclared_header)
      expect( validator.info_messages.size ).to eql(1)
      expect( validator.info_messages.first.type).to eql(:assumed_header)
    end

    it "give undeclared header error if content type is wrong" do
      stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/html"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
      validator = Csvlint::Validator.new("http://example.com/example.csv")
      expect( validator.header? ).to eql(true)
      expect( validator.errors.size ).to eql(2)
      expect( validator.errors[0].type).to eql(:wrong_content_type)
      expect( validator.errors[1].type).to eql(:undeclared_header)
      expect( validator.info_messages.size ).to eql(1)
      expect( validator.info_messages.first.type).to eql(:assumed_header)
    end

  end
  
  context "when validating headers" do
    it "should warn if column names aren't unique" do      
      data = StringIO.new( "minimum, minimum" )
      validator = Csvlint::Validator.new(data)
      expect( validator.validate_header(["minimum", "minimum"]) ).to eql(true)
      expect( validator.warnings.size ).to eql(1)
      expect( validator.warnings.first.type).to eql(:duplicate_column_name)
      expect( validator.warnings.first.category).to eql(:schema)
    end

    it "should warn if column names are blank" do
      data = StringIO.new( "minimum," )
      validator = Csvlint::Validator.new(data)
      
      expect( validator.validate_header(["minimum", ""]) ).to eql(true)
      expect( validator.warnings.size ).to eql(1)
      expect( validator.warnings.first.type).to eql(:empty_column_name)
      expect( validator.warnings.first.category).to eql(:schema)
    end
    
    it "should include info message about missing header when we have assumed a header" do
      data = StringIO.new( "1,2,3\r\n" )
      validator = Csvlint::Validator.new(data)
      
      expect( validator.valid? ).to eql(true)
      expect( validator.info_messages.size ).to eql(1)
      expect( validator.info_messages.first.type).to eql(:assumed_header)
{'fraction_non_alphanumeric': 0.1533370104798676, 'fraction_numerical': 0.004853833425261997, 'mean_word_length': 3.0383073496659243, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6777865', 'n_tokens_mistral': 2918, 'n_tokens_neox': 2862, 'n_words': 606}