<commit_before>module Csvlint

  class Validator

    include Csvlint::ErrorCollector

    attr_reader :encoding, :content_type, :extension, :headers, :line_breaks, :dialect, :csv_header, :schema, :data, :header_processed
    ERROR_MATCHERS = {
        "Missing or stray quote" => :stray_quote,
        "Illegal quoting" => :whitespace,
        "Unclosed quoted field" => :unclosed_quote,
        "Unquoted fields do not allow \\r or \\n" => :line_breaks,
    }

    def initialize(source = nil, dialect = nil, schema = nil, options = {}, row_sep = nil)
      # suggested alternative initialisation parameters: stream, csv_options

      @source = source
      @formats = []
      @schema = schema

      @supplied_dialect = dialect != nil

      @dialect = {
          "header" => true,
          "delimiter" => ",",
          "skipInitialSpace" => true,
          "lineTerminator" => :auto,
          "quoteChar" => '"'
      }.merge(dialect || {})

      @csv_header = @dialect["header"]
      @limit_lines = options[:limit_lines]
      @csv_options = dialect_to_csv_options(@dialect)

      @expected_columns = 0
      @col_counts = []

      @data = [] # it may be advisable to flush this on init?

      reset
      validate
      # TODO - separating the initialise and validate calls means that specs assertions in streaming_validator are more verbose, but can also be unit tested

    end

    def validate
      if @source.class == String
        validate_url
      else
        validate_stream
      end
      finish
    end

    def validate_stream
      i = 1
      @source.each_line do |line|
        break if i == @limit_lines
        validate_line(line, i)
        i = i+1
      end
    end

    def validate_url
      i = 1
      leading = ""
      request = Typhoeus::Request.new(@source, followlocation: true)
      request.on_headers do |response|
        @headers = response.headers
        @content_type = response.headers["content-type"] rescue nil
        @response_code = response.code
        return build_errors(:not_found) if response.code == 404
      end
      request.on_body do |chunk|
        io = StringIO.new(leading + chunk)
        io.each_line do |line|
          break if i == @limit_lines
          # Check if the last line is a line break - in which case it's a full line
          if line[-1, 1].include?("\n")
            validate_line(line, i)
            i = i+1
          else
            # If it's not a full line, then prepare to add it to the beginning of the next chunk
            leading = line
          end
        end
      end
      request.run
      # Validate the last line too
      validate_line(leading, i) unless leading == ""
    end

    def validate_line(input = nil, index = nil)
      single_col = false
      # TODO is single_col still required for single column CSV edge case, see spec it "should work correctly for single columns"
      @stream = input.present? ? input : @stream
      # reassign stream if validate has been invoked with an input, mostly a way of faking loosely coupled stuff while testing
      line = index.present? ? index : 0
      @encoding = input.encoding.to_s
      begin
        report_line_breaks(line)
        parse_contents(@stream, line)
          # rescue CSV::MalformedCSVError => e
          # build_exception_message(e, @stream)
      ensure
        @stream.close if @stream && @stream.respond_to?(:close) #TODO This could get factored into Validate client, or a finishing state in this class
      end
    end

    # analyses the provided csv and builds errors, warnings and info messages
    def parse_contents(stream, line = nil)
      # parse_contents will parse one line and apply headers, formats methods and error handle as appropriate
      current_line = line.present? ? line : 1
      reported_invalid_encoding = false
      all_errors = []

      @csv_options[:encoding] = @encoding

      begin
      row = CSV.parse_line(stream, @csv_options)
        # this is a one line substitute for CSV.new followed by row = CSV.shift. a CSV Row class is required
        # CSV.parse will return an array of arrays which breaks subsequent each_with_index invocations
        # TODO investigate if above would be a drag on memory

      rescue CSV::MalformedCSVError => e
        build_exception_messages(e, stream, current_line)
      end

      @data << row
      # TODO currently it doesn't matter where the above rescue is the @data array is either populated with nil or nothing
      # TODO is that intended behaviour?
      if row
        if current_line <= 1 && @csv_header
          # this conditional should be refactored somewhere
          row = row.reject { |col| col.nil? || col.empty? }
          validate_header(row)
          @col_counts << row.size
        else
          build_formats(row)
          @col_counts << row.reject { |col| col.nil? || col.empty? }.size
          @expected_columns = row.size unless @expected_columns != 0
          build_errors(:blank_rows, :structure, current_line, nil, stream.to_s) if row.reject { |c| c.nil? || c.empty? }.size == 0
          # Builds errors and warnings related to the provided schema file
          if @schema
            @schema.validate_row(row, current_line, all_errors)
            @errors += @schema.errors
            all_errors += @schema.errors
            @warnings += @schema.warnings
          else
            build_errors(:ragged_rows, :structure, current_line, nil, stream.to_s) if !row.empty? && row.size != @expected_columns
          end
        end
      end
      # TODO the below argumenterror is an artefact of when everything was in one long method
      # TODO however this is an important rescue to content parsing as the README stipulates it catches
      # TODO "encoding error when parsing row, e.g. because of invalid characters"
      # rescue ArgumentError => ae
      #   build_errors(:invalid_encoding, :structure, current_line, nil, current_line) unless reported_invalid_encoding
      #   reported_invalid_encoding = true
      # end
    end

    def finish
      sum = @col_counts.inject(:+)
      unless sum.nil?
        build_warnings(:title_row, :structure) if @col_counts.first < (sum / @col_counts.size.to_f)
      end
      # return expected_columns to calling class
      build_warnings(:check_options, :structure) if @expected_columns == 1
      check_consistency
      validate_metadata
    end

    def validate_metadata
      assumed_header = undeclared_header = !@supplied_dialect

      if @headers
        if @headers["content-type"] =~ /text\/csv/
          @csv_header = true
          undeclared_header = false
          assumed_header = true
        end
        if @headers["content-type"] =~ /header=(present|absent)/
          @csv_header = true if $1 == "present"
          @csv_header = false if $1 == "absent"
          undeclared_header = false
          assumed_header = false
        end
        if @headers["content-type"] !~ /charset=/
          build_warnings(:no_encoding, :context)
        else
          build_warnings(:encoding, :context) if @encoding != "UTF-8"
        end
        build_warnings(:no_content_type, :context) if @content_type == nil
        build_warnings(:excel, :context) if @content_type == nil && @extension =~ /.xls(x)?/
        build_errors(:wrong_content_type, :context) unless (@content_type && @content_type =~ /text\/csv/)<commit_msg>Simplify assumed header logic<commit_after>module Csvlint

  class Validator

    include Csvlint::ErrorCollector

    attr_reader :encoding, :content_type, :extension, :headers, :line_breaks, :dialect, :csv_header, :schema, :data, :header_processed
    ERROR_MATCHERS = {
        "Missing or stray quote" => :stray_quote,
        "Illegal quoting" => :whitespace,
        "Unclosed quoted field" => :unclosed_quote,
        "Unquoted fields do not allow \\r or \\n" => :line_breaks,
    }

    def initialize(source = nil, dialect = {}, schema = nil, options = {}, row_sep = nil)
      # suggested alternative initialisation parameters: stream, csv_options

      @source = source
      @formats = []
      @schema = schema

      @assumed_header = dialect["header"].nil?

      @dialect = {
          "header" => true,
          "delimiter" => ",",
          "skipInitialSpace" => true,
          "lineTerminator" => :auto,
          "quoteChar" => '"'
      }.merge(dialect)

      @csv_header = @dialect["header"]
      @limit_lines = options[:limit_lines]
      @csv_options = dialect_to_csv_options(@dialect)

      @expected_columns = 0
      @col_counts = []

      @data = [] # it may be advisable to flush this on init?

      reset
      validate
      # TODO - separating the initialise and validate calls means that specs assertions in streaming_validator are more verbose, but can also be unit tested

    end

    def validate
      if @source.class == String
        validate_url
      else
        validate_stream
      end
      finish
    end

    def validate_stream
      i = 1
      @source.each_line do |line|
        break if i == @limit_lines
        validate_line(line, i)
        i = i+1
      end
    end

    def validate_url
      i = 1
      leading = ""
      request = Typhoeus::Request.new(@source, followlocation: true)
      request.on_headers do |response|
        @headers = response.headers
        @content_type = response.headers["content-type"] rescue nil
        @response_code = response.code
        return build_errors(:not_found) if response.code == 404
      end
      request.on_body do |chunk|
        io = StringIO.new(leading + chunk)
        io.each_line do |line|
          break if i == @limit_lines
          # Check if the last line is a line break - in which case it's a full line
          if line[-1, 1].include?("\n")
            validate_line(line, i)
            i = i+1
          else
            # If it's not a full line, then prepare to add it to the beginning of the next chunk
            leading = line
          end
        end
      end
      request.run
      # Validate the last line too
      validate_line(leading, i) unless leading == ""
    end

    def validate_line(input = nil, index = nil)
      single_col = false
      # TODO is single_col still required for single column CSV edge case, see spec it "should work correctly for single columns"
      @stream = input.present? ? input : @stream
      # reassign stream if validate has been invoked with an input, mostly a way of faking loosely coupled stuff while testing
      line = index.present? ? index : 0
      @encoding = input.encoding.to_s
      begin
        report_line_breaks(line)
        parse_contents(@stream, line)
          # rescue CSV::MalformedCSVError => e
          # build_exception_message(e, @stream)
      ensure
        @stream.close if @stream && @stream.respond_to?(:close) #TODO This could get factored into Validate client, or a finishing state in this class
      end
    end

    # analyses the provided csv and builds errors, warnings and info messages
    def parse_contents(stream, line = nil)
      # parse_contents will parse one line and apply headers, formats methods and error handle as appropriate
      current_line = line.present? ? line : 1
      reported_invalid_encoding = false
      all_errors = []

      @csv_options[:encoding] = @encoding

      begin
      row = CSV.parse_line(stream, @csv_options)
        # this is a one line substitute for CSV.new followed by row = CSV.shift. a CSV Row class is required
        # CSV.parse will return an array of arrays which breaks subsequent each_with_index invocations
        # TODO investigate if above would be a drag on memory

      rescue CSV::MalformedCSVError => e
        build_exception_messages(e, stream, current_line)
      end

      @data << row
      # TODO currently it doesn't matter where the above rescue is the @data array is either populated with nil or nothing
      # TODO is that intended behaviour?
      if row
        if current_line <= 1 && @csv_header
          # this conditional should be refactored somewhere
          row = row.reject { |col| col.nil? || col.empty? }
          validate_header(row)
          @col_counts << row.size
        else
          build_formats(row)
          @col_counts << row.reject { |col| col.nil? || col.empty? }.size
          @expected_columns = row.size unless @expected_columns != 0
          build_errors(:blank_rows, :structure, current_line, nil, stream.to_s) if row.reject { |c| c.nil? || c.empty? }.size == 0
          # Builds errors and warnings related to the provided schema file
          if @schema
            @schema.validate_row(row, current_line, all_errors)
            @errors += @schema.errors
            all_errors += @schema.errors
            @warnings += @schema.warnings
          else
            build_errors(:ragged_rows, :structure, current_line, nil, stream.to_s) if !row.empty? && row.size != @expected_columns
          end
        end
      end
      # TODO the below argumenterror is an artefact of when everything was in one long method
      # TODO however this is an important rescue to content parsing as the README stipulates it catches
      # TODO "encoding error when parsing row, e.g. because of invalid characters"
      # rescue ArgumentError => ae
      #   build_errors(:invalid_encoding, :structure, current_line, nil, current_line) unless reported_invalid_encoding
      #   reported_invalid_encoding = true
      # end
    end

    def finish
      sum = @col_counts.inject(:+)
      unless sum.nil?
        build_warnings(:title_row, :structure) if @col_counts.first < (sum / @col_counts.size.to_f)
      end
      # return expected_columns to calling class
      build_warnings(:check_options, :structure) if @expected_columns == 1
      check_consistency
      validate_metadata
    end

    def validate_metadata

      if @headers
        if @headers["content-type"] =~ /text\/csv/
          @csv_header = true
          undeclared_header = false
          assumed_header = @assumed_header.present?
        end
        if @headers["content-type"] =~ /header=(present|absent)/
          @csv_header = true if $1 == "present"
          @csv_header = false if $1 == "absent"
          undeclared_header = false
          assumed_header = false
        end
        if @headers["content-type"] !~ /charset=/
          build_warnings(:no_encoding, :context)
        else
          build_warnings(:encoding, :context) if @encoding != "UTF-8"
        end
        build_warnings(:no_content_type, :context) if @content_type == nil
        build_warnings(:excel, :context) if @content_type == nil && @extension =~ /.xls(x)?/
        build_errors(:wrong_content_type, :context) unless (@content_type && @content_type =~ /text\/csv/)
{'fraction_non_alphanumeric': 0.08847221282068639, 'fraction_numerical': 0.002572260204426995, 'mean_word_length': 2.383100526677353, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7522026', 'n_tokens_mistral': 4211, 'n_tokens_neox': 4057, 'n_words': 1343}