<commit_before>module Fog
  module AWS
    class Kinesis
      class Real
        # Gets data records from a shard.
        #
        # ==== Options
        # * Limit<~Number>: he maximum number of records to return.
        # * ShardIterator<~String>: The position in the shard from which you want to start sequentially reading data records.
        # ==== Returns
        # * response<~Excon::Response>:
        #
        # ==== See Also
        # https://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html
        #
        def get_records(options={})
          body = {
            "Limit" => options.delete("Limit") || 1,
            "ShardIterator" => options.delete("ShardIterator")
          }.reject{ |_,v| v.nil? }

          response = request({
                               'X-Amz-Target' => "Kinesis_#{@version}.GetRecords",
                               :body          => body,
                             }.merge(options))
          response.body = Fog::JSON.decode(response.body) unless response.body.nil?
          response
        end
      end

      class Mock
        def get_records(options={})
          shard_iterator = Fog::JSON.decode(options.delete("ShardIterator"))
          limit = options.delete("Limit") || 1
          stream_name = shard_iterator["StreamName"]
          shard_id = shard_iterator["ShardId"]
          starting_sequence_number = (shard_iterator["StartingSequenceNumber"] || 1).to_i

          unless stream = data[:kinesis_streams].detect{ |s| s["StreamName"] == stream_name }
            raise 'unknown stream'
          end

          unless shard = stream["Shards"].detect{ |shard| shard["ShardId"] == shard_id }
            raise 'unknown shard'
          end

          records = []
          shard["Records"].each do |record|
            next if record["SequenceNumber"].to_i < starting_sequence_number
            records << record
            break if records.size == limit
          end

          shard_iterator["StartingSequenceNumber"] = if records.empty?
            starting_sequence_number.to_s
          else
            (records.last["SequenceNumber"].to_i + 1).to_s
          end

          response = Excon::Response.new<commit_msg>Remove default limit for get_records<commit_after>module Fog
  module AWS
    class Kinesis
      class Real
        # Gets data records from a shard.
        #
        # ==== Options
        # * Limit<~Number>: The maximum number of records to return.
        # * ShardIterator<~String>: The position in the shard from which you want to start sequentially reading data records.
        # ==== Returns
        # * response<~Excon::Response>:
        #
        # ==== See Also
        # https://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html
        #
        def get_records(options={})
          body = {
            "Limit" => options.delete("Limit"),
            "ShardIterator" => options.delete("ShardIterator")
          }.reject{ |_,v| v.nil? }

          response = request({
                               'X-Amz-Target' => "Kinesis_#{@version}.GetRecords",
                               :body          => body,
                             }.merge(options))
          response.body = Fog::JSON.decode(response.body) unless response.body.nil?
          response
        end
      end

      class Mock
        def get_records(options={})
          shard_iterator = Fog::JSON.decode(options.delete("ShardIterator"))
          limit = options.delete("Limit") || -1
          stream_name = shard_iterator["StreamName"]
          shard_id = shard_iterator["ShardId"]
          starting_sequence_number = (shard_iterator["StartingSequenceNumber"] || 1).to_i

          unless stream = data[:kinesis_streams].detect{ |s| s["StreamName"] == stream_name }
            raise 'unknown stream'
          end

          unless shard = stream["Shards"].detect{ |shard| shard["ShardId"] == shard_id }
            raise 'unknown shard'
          end

          records = []
          shard["Records"].each do |record|
            next if record["SequenceNumber"].to_i < starting_sequence_number
            records << record
            break if records.size == limit
          end

          shard_iterator["StartingSequenceNumber"] = if records.empty?
            starting_sequence_number.to_s
          else
            (records.last["SequenceNumber"].to_i + 1).to_s
          end

          response = Excon::Response.new
{'fraction_non_alphanumeric': 0.11317794710891976, 'fraction_numerical': 0.0015688032272523531, 'mean_word_length': 1.857234314980794, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6529958', 'n_tokens_mistral': 1209, 'n_tokens_neox': 1178, 'n_words': 262}