<commit_before>module Suspension
  # Compares strings and returns diffs
  class StringComparer

    # Compares string_1 with string_2 using diff_match_patch.
    # @param[String] string_1
    # @param[String] string_2
    # @return[Array] An array of diffs like so:
    # [[1, 'added', 'line 42', 'text_before text_after'], [-1, 'removed ', 'line 43', 'text_before removed text_after']]
    # All information is relative to string_1. 1 means a string was added, -1 it was deleted.
    def self.compare(string_1, string_2)
      if string_1 == string_2
        return []
      else
        diffs = Suspension::DiffAlgorithm.new.call(string_1, string_2)
        deltas = []
        char_num = 0
        line_num = 1
        excerpt_window = 20
        # Add context information to diffs
        deltas = diffs.map { |diff|
          begin
            excerpt_start = [(char_num - excerpt_window), 0].max
            excerpt_end = [(char_num + diff.last.length + excerpt_window), string_1.length].min
            excerpt = case diff.first
            when -1
              # use string_1 as context for deletions
              string_1[excerpt_start..excerpt_end]
            when 1
              # use string_2 as context for additions
              string_2[excerpt_start..excerpt_end]
            when 0
              nil
            else
              raise "Handle this: #{ diff.inspect }"
            end
            r = [
              diff.first, # type of modification
              diff.last, # diff string
              "line #{ line_num }",
              excerpt
            ]
            if [0,-1].include?(diff.first)
              # only count chars and newlines in identical or deletions since all info
              # refers to string_1
              char_num += diff.last.length
              line_num += diff.last.count("\n")
            end
            r
          rescue ArgumentError => e
            # Handles invalid UTF-8 byte sequences in diff
            valid_excerpt, valid_string = [excerpt, diff.last].map { |e|
              e.force_encoding('UTF-8') \
               .encode('UTF-16', :invalid => :replace, :replace => '[invalid UTF-8 byte]') \
               .encode('UTF-8')
            }
            $stderr.puts "Error details:"
            $stderr.puts " - line: #{ line_num }"
            $stderr.puts " - diff: #{ valid_string.inspect }"
            $stderr.puts " - excerpt: #{ excerpt.inspect }"
            raise e
          end
        }
        deltas = deltas.find_all { |e| 0 != e.first }
      end
    end

  end
end<commit_msg>Made error report nil resistent<commit_after>module Suspension
  # Compares strings and returns diffs
  class StringComparer

    # Compares string_1 with string_2 using diff_match_patch.
    # @param[String] string_1
    # @param[String] string_2
    # @return[Array] An array of diffs like so:
    # [[1, 'added', 'line 42', 'text_before text_after'], [-1, 'removed ', 'line 43', 'text_before removed text_after']]
    # All information is relative to string_1. 1 means a string was added, -1 it was deleted.
    def self.compare(string_1, string_2)
      if string_1 == string_2
        return []
      else
        diffs = Suspension::DiffAlgorithm.new.call(string_1, string_2)
        deltas = []
        char_num = 0
        line_num = 1
        excerpt_window = 20
        # Add context information to diffs
        deltas = diffs.map { |diff|
          begin
            excerpt_start = [(char_num - excerpt_window), 0].max
            excerpt_end = [(char_num + diff.last.length + excerpt_window), string_1.length].min
            excerpt = case diff.first
            when -1
              # use string_1 as context for deletions
              string_1[excerpt_start..excerpt_end]
            when 1
              # use string_2 as context for additions
              string_2[excerpt_start..excerpt_end]
            when 0
              nil
            else
              raise "Handle this: #{ diff.inspect }"
            end
            r = [
              diff.first, # type of modification
              diff.last, # diff string
              "line #{ line_num }",
              excerpt
            ]
            if [0,-1].include?(diff.first)
              # only count chars and newlines in identical or deletions since all info
              # refers to string_1
              char_num += diff.last.length
              line_num += diff.last.count("\n")
            end
            r
          rescue ArgumentError => e
            # Handles invalid UTF-8 byte sequences in diff
            valid_excerpt, valid_string = [excerpt, diff.last].map { |e|
              e.to_s.force_encoding('UTF-8') \
                    .encode('UTF-16', :invalid => :replace, :replace => '[invalid UTF-8 byte]') \
                    .encode('UTF-8')
            }
            $stderr.puts "Error details:"
            $stderr.puts " - line: #{ line_num }"
            $stderr.puts " - diff: #{ valid_string.inspect }"
            $stderr.puts " - excerpt: #{ excerpt.inspect }"
            raise e
          end
        }
        deltas = deltas.find_all { |e| 0 != e.first }
      end
    end

  end
end
{'fraction_non_alphanumeric': 0.09712299671751304, 'fraction_numerical': 0.016219347364356054, 'mean_word_length': 1.613521695257316, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4605428', 'n_tokens_mistral': 1549, 'n_tokens_neox': 1465, 'n_words': 409}