<commit_before>require 'csv'
require 'stringio'

module OptimusPrime
  module Destinations
    class LocalCsv < Destination
      attr_reader :fields, :append_mode, :file_path

      def initialize(fields:, file_path:, append_mode: false, should_write_header: true, **options)
        @fields = fields
        @file_path = file_path
        @append_mode = append_mode
        @options = options
        @should_write_header = should_write_header

        # always assume that we should not write header if the file already exists and we are appending it
        if @append_mode && File.exist?(@file_path)
          @should_write_header = false
        end

        open_file
      end

      def write(record)
        write_header if @should_write_header
        write_row format record
      end

      def close
        @csv.close
      end

      private

      def open_file
        mode = @append_mode ? 'ab' : 'wb'
        @csv = CSV.open(@file_path, mode, @options)
      end

      def write_header
        write_row fields
        @should_write_header = false
      end

      def write_row(row)
        @csv << row
      end
<commit_msg>Cleanup local csv destination<commit_after>require 'csv'
require 'stringio'

module OptimusPrime
  module Destinations
    class LocalCsv < Destination
      attr_reader :fields, :append_mode, :file_path

      def initialize(fields:, file_path:, append_mode: false, should_write_header: true, **options)
        @fields = fields
        @file_path = file_path
        @append_mode = append_mode
        @options = options
        @should_write_header = should_write_header

        # always assume that we should not write header if the file already
        # exists and we are appending it
        @should_write_header = false if @append_mode && File.exist?(@file_path)

        open_file
      end

      def write(record)
        write_header if @should_write_header
        write_row format record
      end

      def close
        @csv.close
      end

      private

      def open_file
        mode = @append_mode ? 'ab' : 'wb'
        @csv = CSV.open(@file_path, mode, @options)
      end

      def write_header
        write_row fields
        @should_write_header = false
      end

      def write_row(row)
        @csv << row
      end

{'fraction_non_alphanumeric': 0.06570931244560488, 'fraction_numerical': 0.0, 'mean_word_length': 2.219887955182073, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4638727', 'n_tokens_mistral': 672, 'n_tokens_neox': 648, 'n_words': 194}