<commit_before>module Daybreak
  # Writer's handle the actually fiddly task of committing data to disk.
  # They have a Worker instance that writes in a select loop.
  class Writer
    # Open up the file, ready it for binary and nonblocking writing.
    def initialize(file)
      @file = file

      open!

      @worker = Worker.new(@fd)
    end

    # Send a record to the workers queue.
    def write(record)
      @worker.enqueue record
    end

    # Finish writing
    def finish!
      @worker.finish!
    end

    # Flush pending commits, and restart the worker.
    def flush!
      @worker.flush!
    end

    # Finish writing and close the file descriptor.
    def close!
      finish!
      @fd.close
    end

    # Truncate the file.
    def truncate!
      flush!
      @fd.truncate(0)
      open!
    end

    private

    def open!
      @fd = File.open @file, 'a'
      @fd.binmode

      if defined?(Fcntl::O_NONBLOCK)
        f = @fd.fcntl(Fcntl::F_GETFL, 0)
        @fd.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK | f)
      end
    end

    # Workers handle the actual fiddly bits of asynchronous io and
    # and handle background writes.
    class Worker
      include Locking

      def initialize(fd)
        @queue  = Queue.new
        @fd     = fd
        @buffer = ""
        @thread = Thread.new { work }
        at_exit { finish! }
      end

      # Queue up a write to be committed later.
      def enqueue(record)
        @queue << record.representation
      end

      # Loop and block if we don't have work to do or if
      # the file isn't ready for another write just yet.
      def work
        buf = ""
        loop do
          str = @queue.pop
          if str.nil?
            @fd.flush
            break<commit_msg>Remove unused instance variable<commit_after>module Daybreak
  # Writer's handle the actually fiddly task of committing data to disk.
  # They have a Worker instance that writes in a select loop.
  class Writer
    # Open up the file, ready it for binary and nonblocking writing.
    def initialize(file)
      @file = file

      open!

      @worker = Worker.new(@fd)
    end

    # Send a record to the workers queue.
    def write(record)
      @worker.enqueue record
    end

    # Finish writing
    def finish!
      @worker.finish!
    end

    # Flush pending commits, and restart the worker.
    def flush!
      @worker.flush!
    end

    # Finish writing and close the file descriptor.
    def close!
      finish!
      @fd.close
    end

    # Truncate the file.
    def truncate!
      flush!
      @fd.truncate(0)
      open!
    end

    private

    def open!
      @fd = File.open @file, 'a'
      @fd.binmode

      if defined?(Fcntl::O_NONBLOCK)
        f = @fd.fcntl(Fcntl::F_GETFL, 0)
        @fd.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK | f)
      end
    end

    # Workers handle the actual fiddly bits of asynchronous io and
    # and handle background writes.
    class Worker
      include Locking

      def initialize(fd)
        @queue  = Queue.new
        @fd     = fd
        @thread = Thread.new { work }
        at_exit { finish! }
      end

      # Queue up a write to be committed later.
      def enqueue(record)
        @queue << record.representation
      end

      # Loop and block if we don't have work to do or if
      # the file isn't ready for another write just yet.
      def work
        buf = ""
        loop do
          str = @queue.pop
          if str.nil?
            @fd.flush
            break
{'fraction_non_alphanumeric': 0.0746226146397038, 'fraction_numerical': 0.0011392765593847907, 'mean_word_length': 1.7850911974623316, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1166940', 'n_tokens_mistral': 1106, 'n_tokens_neox': 1053, 'n_words': 379}