<commit_before>require 'models/line'
require 'models/section'

module Diffed
  class UnifiedDiffParser
    def initialize(lines)
      @lines, @sections = lines, []
      reset_section!
    end

    def parse!
      raise "Already parsed into #{@sections.length} sections." unless @sections.empty?
      
      @lines.each do |line|
        if HeaderLineParser.is_header?(line)
          @curr_header, @curr_lines = HeaderLineParser.new(line), []
        elsif @curr_header.nil?
          # Do nothing.  We haven't started yet.
          # puts "Ignoring line: #{line}"
        elsif line =~ /\\ No newline at end of file/
          handle_missing_newline
        else
          parse_code_line line
        end
      end      
      
      self # evidence that I'm a Java developer at heart?
    end
    
    def sections
      raise "Not parsed.  Call parse! before attempting to read sections from the parser." if @sections.empty?
      @sections
    end
    
    private
    def handle_missing_newline
      if @curr_lines.empty?
        @sections.last.lines.last.no_newline = true
      else
        @curr_lines.last.no_newline = true
      end      
    end
    
    def parse_code_line(line)
      line_parser = LineParser.new(line)          
      @curr_lines << line_parser.line(@curr_header, @left_counter, @right_counter)
      @left_counter, @right_counter = line_parser.increment(@left_counter, @right_counter)                
      
      if @curr_header.section_complete? @left_counter, @right_counter
        @sections << Section.new(@curr_header.line, @curr_lines)
        reset_section!
      end      
    end
    
    def reset_section!
      @curr_header, @curr_lines, @left_counter, @right_counter = nil, [], 0, 0
    end 
  
    class LineParser
      def initialize(line_text)
        @line_text = line_text
        
        if line_text.start_with? "-"
          @type = :left
        elsif line_text.start_with? "+"
          @type = :right      
        elsif line_text.start_with? " "
          @type = :both               
        else
          raise "Unparseable line: #{line}"
        end
      end
      
      def increment(left_counter, right_counter)
        case @type
        when :left
          [left_counter + 1, right_counter]
        when :right
          [left_counter, right_counter + 1]
        when :both
          [left_counter + 1, right_counter + 1]
        end
      end
      
      def line(header, left_counter, right_counter)
        left = left_line_num header, left_counter
        right = right_line_num header, right_counter
        Line.new(@type, @line_text, left, right)
      end
      
      private
      def left_line_num(header, left_counter)
        @type == :right ? "." : (left_counter + header.line_nums[:left][:from])
      end
      
      def right_line_num(header, right_counter)
        @type == :left ? "." : (right_counter + header.line_nums[:right][:from])
      end
    end
  
    class HeaderLineParser
      attr_reader :line_nums, :text
    
      def initialize(line)
        @text = line
      
        if line =~ /^@@ -(\d+),(\d+) \+(\d+),(\d+) @@/        
          @line_nums = {left: {from: $1.to_i, lines: $2.to_i}, right: {from: $3.to_i, lines: $4.to_i}}
        elsif line =~ /^@@ -(\d+) \+(\d+) @@/
          @line_nums = {left: {from: $1.to_i, lines: $1.to_i}, right: {from: $2.to_i, lines: $2.to_i}}
        else
          raise "Unparseable header line: #{line}"
        end
      
        if @line_nums[:right][:lines] > @line_nums[:left][:lines]
          @side_to_count = :right
        else
          @side_to_count = :left
        end      
      end
    
      def self.is_header?(line)
        line.start_with? "@@ "
      end
    
      def section_complete?(left_count, right_count)
        left_count >= @line_nums[:left][:lines] && right_count >= @line_nums[:right][:lines]
      end
      
      def line
        @text
      end
    end  
  end
end<commit_msg>Shouldn't throw an error if sections is empty.  If there are no diffs in the output, then parse! will leave an empty sections array, which is fine.<commit_after>require 'models/line'
require 'models/section'

module Diffed
  class UnifiedDiffParser
    def initialize(lines)
      @lines, @sections = lines, []
      reset_section!
    end

    def parse!
      raise "Already parsed into #{@sections.length} sections." unless @sections.empty?
      
      @lines.each do |line|
        if HeaderLineParser.is_header?(line)
          @curr_header, @curr_lines = HeaderLineParser.new(line), []
        elsif @curr_header.nil?
          # Do nothing.  We haven't started yet.
          # puts "Ignoring line: #{line}"
        elsif line =~ /\\ No newline at end of file/
          handle_missing_newline
        else
          parse_code_line line
        end
      end      
      
      self # evidence that I'm a Java developer at heart?
    end
    
    def sections
      @sections
    end
    
    private
    def handle_missing_newline
      if @curr_lines.empty?
        @sections.last.lines.last.no_newline = true
      else
        @curr_lines.last.no_newline = true
      end      
    end
    
    def parse_code_line(line)
      line_parser = LineParser.new(line)          
      @curr_lines << line_parser.line(@curr_header, @left_counter, @right_counter)
      @left_counter, @right_counter = line_parser.increment(@left_counter, @right_counter)                
      
      if @curr_header.section_complete? @left_counter, @right_counter
        @sections << Section.new(@curr_header.line, @curr_lines)
        reset_section!
      end      
    end
    
    def reset_section!
      @curr_header, @curr_lines, @left_counter, @right_counter = nil, [], 0, 0
    end 
  
    class LineParser
      def initialize(line_text)
        @line_text = line_text
        
        if line_text.start_with? "-"
          @type = :left
        elsif line_text.start_with? "+"
          @type = :right      
        elsif line_text.start_with? " "
          @type = :both               
        else
          raise "Unparseable line: #{line}"
        end
      end
      
      def increment(left_counter, right_counter)
        case @type
        when :left
          [left_counter + 1, right_counter]
        when :right
          [left_counter, right_counter + 1]
        when :both
          [left_counter + 1, right_counter + 1]
        end
      end
      
      def line(header, left_counter, right_counter)
        left = left_line_num header, left_counter
        right = right_line_num header, right_counter
        Line.new(@type, @line_text, left, right)
      end
      
      private
      def left_line_num(header, left_counter)
        @type == :right ? "." : (left_counter + header.line_nums[:left][:from])
      end
      
      def right_line_num(header, right_counter)
        @type == :left ? "." : (right_counter + header.line_nums[:right][:from])
      end
    end
  
    class HeaderLineParser
      attr_reader :line_nums, :text
    
      def initialize(line)
        @text = line
      
        if line =~ /^@@ -(\d+),(\d+) \+(\d+),(\d+) @@/        
          @line_nums = {left: {from: $1.to_i, lines: $2.to_i}, right: {from: $3.to_i, lines: $4.to_i}}
        elsif line =~ /^@@ -(\d+) \+(\d+) @@/
          @line_nums = {left: {from: $1.to_i, lines: $1.to_i}, right: {from: $2.to_i, lines: $2.to_i}}
        else
          raise "Unparseable header line: #{line}"
        end
      
        if @line_nums[:right][:lines] > @line_nums[:left][:lines]
          @side_to_count = :right
        else
          @side_to_count = :left
        end      
      end
    
      def self.is_header?(line)
        line.start_with? "@@ "
      end
    
      def section_complete?(left_count, right_count)
        left_count >= @line_nums[:left][:lines] && right_count >= @line_nums[:right][:lines]
      end
      
      def line
        @text
      end
    end  
  end
end
{'fraction_non_alphanumeric': 0.10880763915064706, 'fraction_numerical': 0.003518029903254178, 'mean_word_length': 1.901932191031717, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2934984', 'n_tokens_mistral': 2540, 'n_tokens_neox': 2500, 'n_words': 590}