<commit_before>module YAYEnc
  class Part
    attr_accessor :line_width, :name, :pcrc32, :crc32
    attr_accessor :part_size, :total_size
    attr_accessor :part_num, :part_total
    attr_accessor :start_byte, :end_byte

    def initialize
      @lines = []
      @pcrc32 = 0
      @crc32 = 0
    end

    def <<(line)
      @lines << line
    end

    def data
      @lines.join("\n")
    end

    def final_part?
      part_num == part_total
    end
    
    def multi_part?
      part_total > 1
    end

    def to_s
      to_io.read
    end

    def to_io
      StringIO.new.tap do |sio|
        sio.write(ybegin << "\n")
        sio.write(ypart << "\n") if multi_part?
        sio.write(data << "\n")
        sio.write(yend << "\n")
        sio.rewind
      end
    end<commit_msg>[Part] use \r\n for newline<commit_after>module YAYEnc
  class Part
    attr_accessor :line_width, :name, :pcrc32, :crc32
    attr_accessor :part_size, :total_size
    attr_accessor :part_num, :part_total
    attr_accessor :start_byte, :end_byte

    def initialize
      @lines = []
      @pcrc32 = 0
      @crc32 = 0
    end

    def <<(line)
      @lines << line
    end

    def data
      @lines.join("\n")
    end

    def final_part?
      part_num == part_total
    end
    
    def multi_part?
      part_total > 1
    end

    def to_s
      to_io.read
    end

    def to_io
      StringIO.new.tap do |sio|
        sio.write(ybegin << "\r\n")
        sio.write(ypart << "\r\n") if multi_part?
        sio.write(data << "\r\n")
        sio.write(yend << "\r\n")
        sio.rewind
      end
    end
{'fraction_non_alphanumeric': 0.11166875784190715, 'fraction_numerical': 0.013801756587202008, 'mean_word_length': 1.83303730017762, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1830309', 'n_tokens_mistral': 621, 'n_tokens_neox': 591, 'n_words': 137}