<commit_before>module SugarCRM; module FinderMethods
  module ClassMethods
    private 
      def find_initial(options)
        options.update(:limit => 1)
        result = find_by_sql(options)
        return result.first if result.instance_of? Array # find_by_sql will return an Array if result are found
        result
      end
  
      def find_from_ids(ids, options, &block)
        expects_array = ids.first.kind_of?(Array)
        return ids.first if expects_array && ids.first.empty?

        ids = ids.flatten.compact.uniq

        case ids.size
          when 0
            raise RecordNotFound, "Couldn't find #{self._module.name} without an ID"
          when 1
            result = find_one(ids.first, options)
            expects_array ? [ result ] : result
          else
            find_some(ids, options, &block)
        end
      end
  
      def find_one(id, options)
      
        if result = connection.get_entry(self._module.name, id, {:fields => self._module.fields.keys})
          result
        else
          raise RecordNotFound, "Couldn't find #{name} with ID=#{id}#{conditions}"
        end
      end
    
      def find_some(ids, options, &block)
        result = connection.get_entries(self._module.name, ids, {:fields => self._module.fields.keys})

        # Determine expected size from limit and offset, not just ids.size.
        expected_size =
          if options[:limit] && ids.size > options[:limit]
            options[:limit]
          else
            ids.size
          end

        # 11 ids with limit 3, offset 9 should give 2 results.
        if options[:offset] && (ids.size - options[:offset] < expected_size)
          expected_size = ids.size - options[:offset]
        end

        if result.size == expected_size
          if block_given?
            result.each{|r|
              yield r
            }
          end
          result
        else
          raise RecordNotFound, "Couldn't find all #{name.pluralize} with IDs (#{ids_list})#{conditions} (found #{result.size} results, but was looking for #{expected_size})"
        end
      end
    
      def find_every(options, &block)
        find_by_sql(options, &block)
      end
      
      # the number of records we retrieve with each query
      # it is kept small to avoid timeout issues
      SLICE_SIZE = 5
      SLICE_SIZE.freeze
      # results accumulator stores the results we have fetched so far, recursively
      def find_by_sql(options, results_accumulator=nil, &block)
        # SugarCRM REST API has a bug where, when :limit and :offset options are passed simultaneously,
        # :limit is considered to be the smallest of the two, and :offset is the larger
        # In addition to allowing querying of large datasets while avoiding timeouts (by fetching results in small slices),
        # this implementation fixes the :limit - :offset bug so that it behaves correctly
        
        offset = options[:offset].to_i >= 1 ? options[:offset].to_i : nil
        
        # if many results are requested (i.e. multiple result slices), we call this function recursively
        # this array keeps track of which slice we are retrieving (by updating the :offset and :limit options)
        local_options = {}
        # ensure results are ordered so :limit and :offset option behave in a deterministic fashion
        local_options[:order_by] = :id unless options[:order_by]
        
        # we must ensure limit <= offset (due to bug mentioned above)
        if offset
          local_options[:limit] = [offset.to_i, SLICE_SIZE].min
          local_options[:offset] = offset if offset
        else
          local_options[:limit] = options[:limit] ? [options[:limit].to_i, SLICE_SIZE].min : SLICE_SIZE
        end
        local_options[:limit] = [local_options[:limit], options[:limit]].min if options[:limit] # don't retrieve more records than required
        local_options = options.merge(local_options)
        
        query = query_from_options(local_options)
        result_slice = connection.get_entry_list(self._module.name, query, local_options)
        return results_accumulator unless result_slice
        
        result_slice_array = Array.wrap(result_slice)
        result_slice_array.each{|r| yield r } if block_given?
        results_accumulator = [] unless results_accumulator
        results_accumulator = results_accumulator.concat(result_slice_array)
        
        # adjust options to take into account records that were already retrieved
        updated_options = {:offset => options[:offset].to_i + result_slice_array.size}
        updated_options[:limit] = (options[:limit] ? options[:limit] - result_slice_array.size : nil)
        updated_options = options.merge(updated_options)
        
        # have we retrieved all the records?
        if (updated_options[:limit] && updated_options[:limit] < 1) || local_options[:limit] > result_slice_array.size
          return results_accumulator<commit_msg>in find_by_sql, no need to accumulate records if we're yielding them to a block<commit_after>module SugarCRM; module FinderMethods
  module ClassMethods
    private 
      def find_initial(options)
        options.update(:limit => 1)
        result = find_by_sql(options)
        return result.first if result.instance_of? Array # find_by_sql will return an Array if result are found
        result
      end
  
      def find_from_ids(ids, options, &block)
        expects_array = ids.first.kind_of?(Array)
        return ids.first if expects_array && ids.first.empty?

        ids = ids.flatten.compact.uniq

        case ids.size
          when 0
            raise RecordNotFound, "Couldn't find #{self._module.name} without an ID"
          when 1
            result = find_one(ids.first, options)
            expects_array ? [ result ] : result
          else
            find_some(ids, options, &block)
        end
      end
  
      def find_one(id, options)
      
        if result = connection.get_entry(self._module.name, id, {:fields => self._module.fields.keys})
          result
        else
          raise RecordNotFound, "Couldn't find #{name} with ID=#{id}#{conditions}"
        end
      end
    
      def find_some(ids, options, &block)
        result = connection.get_entries(self._module.name, ids, {:fields => self._module.fields.keys})

        # Determine expected size from limit and offset, not just ids.size.
        expected_size =
          if options[:limit] && ids.size > options[:limit]
            options[:limit]
          else
            ids.size
          end

        # 11 ids with limit 3, offset 9 should give 2 results.
        if options[:offset] && (ids.size - options[:offset] < expected_size)
          expected_size = ids.size - options[:offset]
        end

        if result.size == expected_size
          if block_given?
            result.each{|r|
              yield r
            }
          end
          result
        else
          raise RecordNotFound, "Couldn't find all #{name.pluralize} with IDs (#{ids_list})#{conditions} (found #{result.size} results, but was looking for #{expected_size})"
        end
      end
    
      def find_every(options, &block)
        find_by_sql(options, &block)
      end
      
      # the number of records we retrieve with each query
      # it is kept small to avoid timeout issues
      SLICE_SIZE = 5
      SLICE_SIZE.freeze
      # results accumulator stores the results we have fetched so far, recursively
      def find_by_sql(options, results_accumulator=nil, &block)
        # SugarCRM REST API has a bug where, when :limit and :offset options are passed simultaneously,
        # :limit is considered to be the smallest of the two, and :offset is the larger
        # In addition to allowing querying of large datasets while avoiding timeouts (by fetching results in small slices),
        # this implementation fixes the :limit - :offset bug so that it behaves correctly
        
        offset = options[:offset].to_i >= 1 ? options[:offset].to_i : nil
        
        # if many results are requested (i.e. multiple result slices), we call this function recursively
        # this array keeps track of which slice we are retrieving (by updating the :offset and :limit options)
        local_options = {}
        # ensure results are ordered so :limit and :offset option behave in a deterministic fashion
        local_options[:order_by] = :id unless options[:order_by]
        
        # we must ensure limit <= offset (due to bug mentioned above)
        if offset
          local_options[:limit] = [offset.to_i, SLICE_SIZE].min
          local_options[:offset] = offset if offset
        else
          local_options[:limit] = options[:limit] ? [options[:limit].to_i, SLICE_SIZE].min : SLICE_SIZE
        end
        local_options[:limit] = [local_options[:limit], options[:limit]].min if options[:limit] # don't retrieve more records than required
        local_options = options.merge(local_options)
        
        query = query_from_options(local_options)
        result_slice = connection.get_entry_list(self._module.name, query, local_options)
        return results_accumulator unless result_slice
        
        result_slice_array = Array.wrap(result_slice)
        if block_given?
          result_slice_array.each{|r| yield r }
        else
          results_accumulator = [] unless results_accumulator
          results_accumulator = results_accumulator.concat(result_slice_array)
        end
        
        # adjust options to take into account records that were already retrieved
        updated_options = {:offset => options[:offset].to_i + result_slice_array.size}
        updated_options[:limit] = (options[:limit] ? options[:limit] - result_slice_array.size : nil)
        updated_options = options.merge(updated_options)
        
        # have we retrieved all the records?
        if (updated_options[:limit] && updated_options[:limit] < 1) || local_options[:limit] > result_slice_array.size
          return results_accumulator
{'fraction_non_alphanumeric': 0.08157973471626608, 'fraction_numerical': 0.002194075994814002, 'mean_word_length': 2.410884353741497, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3001237', 'n_tokens_mistral': 2717, 'n_tokens_neox': 2629, 'n_words': 850}