<commit_before>#This class buffers a lot of queries and flushes them out via transactions.
class Knj::Db::Query_buffer
  #Constructor. Takes arguments to be used and a block.
  def initialize(args)
    @args = args
    @queries = []
    @inserts = {}
    @queries_count = 0
    @debug = @args[:debug]
    @lock = Mutex.new
    
    STDOUT.puts "Query buffer started." if @debug
    
    begin
      yield(self)
    ensure
      self.flush
    end
  end
  
  #Adds a query to the buffer.
  def query(str)
    @lock.synchronize do
      STDOUT.print "Adding to buffer: #{str}\n" if @debug
      @queries << str
      @queries_count += 1
    end
    
    self.flush if @queries_count >= 1000
    return nil
  end
  
  #Delete as on a normal Knj::Db.
  #===Examples
  # db.q_buffer do |buffer|
  #   buffer.delete(:users, {:id => 5})
  # end
  def delete(table, where)
    STDOUT.puts "Delete called on table #{table} with arguments: '#{where}'." if @debug
    self.query(@args[:db].delete(table, where, :return_sql => true))
    return nil
  end
  
  #Plans to inset a hash into a table. It will only be inserted when flush is called.
  #===Examples
  # db.q_buffer do |buffer|
  #   buffer.insert(:users, {:name => "<NAME>"})
  # end
  def insert(table, data)
    @lock.synchronize do
      @inserts[table] = [] if !@inserts.key?(table)
      @inserts[table] << data
      @queries_count += 1
    end
    
    self.flush if @queries_count >= 1000
    return nil
  end
  
  #Flushes all queries out in a transaction. This will automatically be called for every 1000 queries.
  def flush
    return nil if @queries_count <= 0
    
    @lock.synchronize do
      @args[:db].transaction do
        @queries.shift(1000).each do |str|
          STDOUT.print "Executing via buffer: #{str}\n" if @debug
          @args[:db].q(str)
        end
        
        @inserts.each do |table, datas_arr|
          while !datas_arr.empty?
            datas_chunk_arr = datas_arr.shift(1000)
            @args[:db].insert_multi(table, datas_chunk_arr)
          end
        end
      end
      
      @inserts.clear
      @queries_count = 0
    end
    
    return nil
  end
end<commit_msg>Now works without a block is given for various reasons.<commit_after>#This class buffers a lot of queries and flushes them out via transactions.
class Knj::Db::Query_buffer
  #Constructor. Takes arguments to be used and a block.
  def initialize(args)
    @args = args
    @queries = []
    @inserts = {}
    @queries_count = 0
    @debug = @args[:debug]
    @lock = Mutex.new
    
    STDOUT.puts "Query buffer started." if @debug
    
    if block_given?
			begin
				yield(self)
			ensure
				self.flush
			end
		end
  end
  
  #Adds a query to the buffer.
  def query(str)
    @lock.synchronize do
      STDOUT.print "Adding to buffer: #{str}\n" if @debug
      @queries << str
      @queries_count += 1
    end
    
    self.flush if @queries_count >= 1000
    return nil
  end
  
  #Delete as on a normal Knj::Db.
  #===Examples
  # db.q_buffer do |buffer|
  #   buffer.delete(:users, {:id => 5})
  # end
  def delete(table, where)
    STDOUT.puts "Delete called on table #{table} with arguments: '#{where}'." if @debug
    self.query(@args[:db].delete(table, where, :return_sql => true))
    return nil
  end
  
  #Plans to inset a hash into a table. It will only be inserted when flush is called.
  #===Examples
  # db.q_buffer do |buffer|
  #   buffer.insert(:users, {:name => "<NAME>"})
  # end
  def insert(table, data)
    @lock.synchronize do
      @inserts[table] = [] if !@inserts.key?(table)
      @inserts[table] << data
      @queries_count += 1
    end
    
    self.flush if @queries_count >= 1000
    return nil
  end
  
  #Flushes all queries out in a transaction. This will automatically be called for every 1000 queries.
  def flush
    return nil if @queries_count <= 0
    
    @lock.synchronize do
      @args[:db].transaction do
        @queries.shift(1000).each do |str|
          STDOUT.print "Executing via buffer: #{str}\n" if @debug
          @args[:db].q(str)
        end
        
        @inserts.each do |table, datas_arr|
          while !datas_arr.empty?
            datas_chunk_arr = datas_arr.shift(1000)
            @args[:db].insert_multi(table, datas_chunk_arr)
          end
        end
      end
      
      @inserts.clear
      @queries_count = 0
    end
    
    return nil
  end
end
{'fraction_non_alphanumeric': 0.10952056350829358, 'fraction_numerical': 0.011815496478073164, 'mean_word_length': 2.302325581395349, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7123618', 'n_tokens_mistral': 1529, 'n_tokens_neox': 1437, 'n_words': 445}