<commit_before>require 'diff/lcs'
require 'diff/lcs/hunk'
require 'pp'

module RSpec
  module Expectations
    class Differ
      # This is snagged from diff/lcs/ldiff.rb (which is a commandline tool)
      def diff_as_string(input_data_new, input_data_old)
        output = matching_encoding("", input_data_old)
        data_old = input_data_old.split(matching_encoding("\n", input_data_old)).map! { |e| e.chomp }
        data_new = input_data_new.split(matching_encoding("\n", input_data_new)).map! { |e| e.chomp }
        diffs = Diff::LCS.diff(data_old, data_new)
        return output if diffs.empty?
        oldhunk = hunk = nil
        file_length_difference = 0
        diffs.each do |piece|
          begin
            hunk = Diff::LCS::Hunk.new(
              data_old, data_new, piece, context_lines, file_length_difference
            )
            file_length_difference = hunk.file_length_difference
            next unless oldhunk
            # Hunks may overlap, which is why we need to be careful when our
            # diff includes lines of context. Otherwise, we might print
            # redundant lines.
            if (context_lines > 0) and hunk.overlaps?(oldhunk)
              if hunk.respond_to?(:merge)
                # diff-lcs 1.2.x
                hunk.merge(oldhunk)
              else
                # diff-lcs 1.1.3
                hunk.unshift(oldhunk)
              end
            else
              output << matching_encoding(oldhunk.diff(format).to_s, output)
            end
          ensure
            oldhunk = hunk
            output << matching_encoding("\n", output)
          end
        end
        #Handle the last remaining hunk
        output << matching_encoding(oldhunk.diff(format).to_s,output)
        output << matching_encoding("\n",output)
        color_diff output
      rescue Encoding::CompatibilityError
        if input_data_new.encoding != input_data_old.encoding
          "Could not produce a diff because the encoding of the actual string (#{input_data_old.encoding}) "+
          "differs from the encoding of the expected string (#{input_data_new.encoding})"
        else
          "Could not produce a diff because of the encoding of the string (#{input_data_old.encoding})"
        end
      end

      def diff_as_object(actual, expected)
        actual_as_string = object_to_string(actual)
        expected_as_string = object_to_string(expected)
        if diff = diff_as_string(actual_as_string, expected_as_string)
          color_diff diff
        end
      end

    protected

      def format
        :unified
      end

      def context_lines
        3
      end

      def color(text, color_code)
        "\e[#{color_code}m#{text}\e[0m"
      end

      def red(text)
        color(text, 31)
      end

      def green(text)
        color(text, 32)
      end

      def blue(text)
        color(text, 34)
      end<commit_msg>Extracting methods on the differ<commit_after>require 'diff/lcs'
require 'diff/lcs/hunk'
require 'pp'

module RSpec
  module Expectations
    class Differ
      def diff_as_string(actual, expected)
        @actual = actual
        @expected = expected

        output = matching_encoding("", expected)

        return output if diffs.empty?
        oldhunk = hunk = nil
        file_length_difference = 0
        diffs.each do |piece|
          begin
            hunk = Diff::LCS::Hunk.new(
              expected_lines, actual_lines, piece, context_lines, file_length_difference
            )
            file_length_difference = hunk.file_length_difference
            next unless oldhunk
            # Hunks may overlap, which is why we need to be careful when our
            # diff includes lines of context. Otherwise, we might print
            # redundant lines.
            if (context_lines > 0) and hunk.overlaps?(oldhunk)
              if hunk.respond_to?(:merge)
                # diff-lcs 1.2.x
                hunk.merge(oldhunk)
              else
                # diff-lcs 1.1.3
                hunk.unshift(oldhunk)
              end
            else
              output << matching_encoding(oldhunk.diff(format).to_s, output)
            end
          ensure
            oldhunk = hunk
            output << matching_encoding("\n", output)
          end
        end
        #Handle the last remaining hunk
        output << matching_encoding(oldhunk.diff(format).to_s,output)
        output << matching_encoding("\n",output)
        color_diff output
      rescue Encoding::CompatibilityError
        if actual.encoding != expected.encoding
          "Could not produce a diff because the encoding of the actual string (#{expected.encoding}) "+
          "differs from the encoding of the expected string (#{actual.encoding})"
        else
          "Could not produce a diff because of the encoding of the string (#{expected.encoding})"
        end
      end

      def diff_as_object(actual, expected)
        actual_as_string = object_to_string(actual)
        expected_as_string = object_to_string(expected)
        if diff = diff_as_string(actual_as_string, expected_as_string)
          color_diff diff
        end
      end

    private

      attr_reader :expected, :actual

      def diffs
        Diff::LCS.diff(expected_lines, actual_lines)
      end

      def expected_lines
        expected.split(matching_encoding("\n", expected)).map! { |e| e.chomp }
      end

      def actual_lines
        actual.split(matching_encoding("\n", actual)).map! { |e| e.chomp }
      end

      def format
        :unified
      end

      def context_lines
        3
      end

      def color(text, color_code)
        "\e[#{color_code}m#{text}\e[0m"
      end

      def red(text)
        color(text, 31)
      end

      def green(text)
        color(text, 32)
      end

      def blue(text)
        color(text, 34)
      end
{'fraction_non_alphanumeric': 0.0795357569551118, 'fraction_numerical': 0.005120327700972862, 'mean_word_length': 1.9197807673143996, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '104441', 'n_tokens_mistral': 1723, 'n_tokens_neox': 1661, 'n_words': 438}