<commit_before>module SimpleRecord

    #
    # We need to make this behave as if the full set were loaded into the array.
    class ResultsArray
        include Enumerable

        attr_reader :next_token, :clz, :params, :items, :i


        def initialize(clz=nil, params=[], items=[], next_token=nil)
            @clz = clz
            #puts 'class=' + clz.inspect
            @params = params
            @items = items
            @next_token = next_token
            @i = 0
        end

        def << (val)
            @items << val
        end

        def [](*i)
            # todo: load items up to i if size > i
            @items[*i]
        end

        def first
            @items[0]
        end

        def last
            @items[@items.length-1]
        end

        def empty?
            @items.empty?
        end

        def include?(obj)
            @items.include?(obj)
        end

        def size
            # puts 'SIZE count=' + @count.inspect
            return @count if @count
            params_for_count = params.dup
            params_for_count[0] = :count
            #puts 'params_for_count=' + params_for_count.inspect
            @count = clz.find(*params_for_count)
            # puts '@count=' + @count.to_s
            @count
        end

        def length
            return size
        end

        def each(&blk)
            limit = nil
            if params.size > 1
                options = params[1]
                limit = options[:limit]
            else
                options = {}
                params[1] = options
            end

            @items.each do |v|
                #puts @i.to_s
                yield v
                @i += 1
                if !limit.nil? && @i >= limit
                    return
                end
            end
            # no more items, but is there a next token?
            return if clz.nil?

            unless next_token.nil?
                #puts 'finding more items...'
                #puts 'params in block=' + params.inspect
                options[:next_token] = next_token
                res = clz.find(*params)
                items = res.items # get the real items array from the ResultsArray
                items.each do |item|
                    @items << item
                end
                @next_token = res.next_token
                each(&blk)
            end
        end

        def delete(item)
            @items.delete(item)
        end

        def delete_at(index)
            @items.delete_at(index)
        end

    end
end
<commit_msg>* CRITICAL BUG IN RESULTS ARRAY fixed<commit_after>module SimpleRecord

    #
    # We need to make this behave as if the full set were loaded into the array.
    class ResultsArray
        include Enumerable

        attr_reader :next_token, :clz, :params, :items, :i


        def initialize(clz=nil, params=[], items=[], next_token=nil)
            @clz = clz
            #puts 'class=' + clz.inspect
            @params = params
            @items = items
            @currentset_items = items
            @next_token = next_token
            @i = 0
        end

        def << (val)
            @items << val
        end

        def [](*i)
            # todo: load items up to i if size > i
            @items[*i]
        end

        def first
            @items[0]
        end

        def last
            @items[@items.length-1]
        end

        def empty?
            @items.empty?
        end

        def include?(obj)
            @items.include?(obj)
        end

        def size
            # puts 'SIZE count=' + @count.inspect
            return @count if @count
            params_for_count = params.dup
            params_for_count[0] = :count
            #puts 'params_for_count=' + params_for_count.inspect
            @count = clz.find(*params_for_count)
            # puts '@count=' + @count.to_s
            @count
        end

        def length
            return size
        end

        def each(&blk)
            limit = nil
            if params.size > 1
                options = params[1]
                limit = options[:limit]
            else
                options = {}
                params[1] = options
            end

            @currentset_items.each do |v|
                #puts @i.to_s
                yield v
                @i += 1
                if !limit.nil? && @i >= limit
                    return
                end
            end
            # no more items, but is there a next token?
            return if clz.nil?

            unless next_token.nil?
                #puts 'finding more items...'
                #puts 'params in block=' + params.inspect
                #puts "i from results_array = " + @i.to_s

                options[:next_token] = next_token
                res = clz.find(*params)
                @currentset_items = res.items # get the real items array from the ResultsArray
                @currentset_items.each do |item|
                    @items << item
                end
                @next_token = res.next_token
                each(&blk)
            end
        end

        def delete(item)
            @items.delete(item)
        end

        def delete_at(index)
            @items.delete_at(index)
        end

    end
end
{'fraction_non_alphanumeric': 0.08218661639962299, 'fraction_numerical': 0.0030160226201696515, 'mean_word_length': 1.094749309119621, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6303243', 'n_tokens_mistral': 1478, 'n_tokens_neox': 1392, 'n_words': 413}