<commit_before>require 'spec_helper'
describe Dyph3::Differ do
  two_way_differs.each do |diff2|
    describe diff2 do
      describe ".merge_two_way_diff" do
        it "show all no changes" do
          t1 = "a b c d".split
          diff = Dyph3::Differ.merge_two_way_diff(t1, t1, diff2: diff2)
          expect(diff.map(&:class)).to eq [Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange]
        end
        # it "should show an add" do
        #   t1  = %w[ White Grey Pink Red Brown Orange Yellow Cyan Blue Violet Invisible ]
        #   t2 = %w[ White Grey Pink Invisible Red Brown Orange Yellow Cyan Blue Violet ]
        #   diff = Dyph3::Differ.merge_two_way_diff(t1, t2, diff2: diff2)
        #   binding.pry
        #   #expect(diff.map(&:class)).to eq [Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::Add]
        # end
        it "should show an add" do
          t1 = "a b c d".split
          t2 = "a b c d e".split
          diff = Dyph3::Differ.merge_two_way_diff(t1, t2, diff2: diff2)
          expect(diff.map(&:class)).to eq [Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::Add]
        end

        it "should show a delete" do
          t1 = "a b c d".split
          t2 = "a b c".split
          diff = Dyph3::Differ.merge_two_way_diff(t1, t2, diff2: diff2)
          expect(diff.map(&:class)).to eq [Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::Delete]
        end

        it "should show a change" do
          t1 = "a b c d".split
          t2 = "a b z d".split
          diff = Dyph3::Differ.merge_two_way_diff(t1, t2, diff2: diff2)
          expect(diff.map(&:class)).to eq [Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::Delete, Dyph3::Action::Add, Dyph3::Action::NoChange]
        end

        it "should work with this real world complex example" do
          t1 = ["#<Mido::TextBlock::Id id=\"atom_12225_brief_p_0a5374c2bb74790ed958dfb8516a76fc_0\">", "#<Mido::TextBlock::Type name=:p>", ":\"Human \"", ":\"activities \"", ":\"likely \"", ":\"caused \"", ":\"the \"", ":\"Holocene \"", ":\"mass \"", ":extinctions", ":\", \"", ":\"and \"", ":\"many \"", ":\"methods \"", ":\"have \"", ":\"been \"", ":\"employed \"", ":\"to \"", ":\"estimate \"", ":\"these \"", ":\"extinction \"", ":rates", ":\".\"", "#<Mido::TextBlock::Type closed=true name=:p>", "#<Mido::TextBlock::Id closed=true id=\"atom_12225_brief_p_0a5374c2bb74790ed958dfb8516a76fc_0\">"]
          t2 = ["#<Mido::TextBlock::Id id=\"atom_12225_brief_p_27e015d07debd946ef199ab53ce1ca29_0\">", "#<Mido::TextBlock::Type name=:p>", ":\"Human \"", ":\"activities \"", ":\"probably \"", ":\"caused \"", ":\"the \"", ":\"Holocene \"", ":\"mass \"", ":extinctions", ":\"; \"", ":\"many \"", ":\"methods \"", ":\"have \"", ":\"been \"", ":\"employed \"", ":\"to \"", ":\"estimate \"", ":\"these \"", ":\"extinction \"", ":rates", ":\".\"", "#<Mido::TextBlock::Type closed=true name=:p>", "#<Mido::TextBlock::Id closed=true id=\"atom_12225_brief_p_27e015d07debd946ef199ab53ce1ca29_0\">"]
          diff = Dyph3::Differ.merge_two_way_diff(t1, t2, diff2: diff2)

          expect(diff.map(&:class)).to eq [ Dyph3::Action::Delete, Dyph3::Action::Add, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::Delete, Dyph3::Action::Add, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::Delete, Dyph3::Action::Delete, Dyph3::Action::Add, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::Delete, Dyph3::Action::Add ]
        end
      end
    end
  end
end<commit_msg>remove legacy mido stuff from a test<commit_after>require 'spec_helper'
describe Dyph3::Differ do
  two_way_differs.each do |diff2|
    describe diff2 do
      describe ".merge_two_way_diff" do
        it "show all no changes" do
          t1 = "a b c d".split
          diff = Dyph3::Differ.merge_two_way_diff(t1, t1, diff2: diff2)
          expect(diff.map(&:class)).to eq [Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange]
        end
       
        it "should show an add" do
          t1 = "a b c d".split
          t2 = "a b c d e".split
          diff = Dyph3::Differ.merge_two_way_diff(t1, t2, diff2: diff2)
          expect(diff.map(&:class)).to eq [Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::Add]
        end

        it "should show a delete" do
          t1 = "a b c d".split
          t2 = "a b c".split
          diff = Dyph3::Differ.merge_two_way_diff(t1, t2, diff2: diff2)
          expect(diff.map(&:class)).to eq [Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::Delete]
        end

        it "should show a change" do
          t1 = "a b c d".split
          t2 = "a b z d".split
          diff = Dyph3::Differ.merge_two_way_diff(t1, t2, diff2: diff2)
          expect(diff.map(&:class)).to eq [Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::Delete, Dyph3::Action::Add, Dyph3::Action::NoChange]
        end

        it "should work with this real world complex example" do
          t1 = ["t1_001", "s1", "s2", "s3", "t1_002", "s4", "s5", "s6", "s7", "s8", "t1_003", "t1_004", "s9", "s10", "s11", "s12", "s13", "s14", "s15", "s16", "s17", "s18", "s19", "s20", "t1_005"]
          t2 = ["t2_001", "s1", "s2", "s3", "t2_002", "s4", "s5", "s6", "s7", "s8", "t2_003", "s9", "s10", "s11", "s12", "s13", "s14", "s15", "s16", "s17", "s18", "s19", "s20", "t2_005"]
          diff = Dyph3::Differ.merge_two_way_diff(t1, t2, diff2: diff2)

          expect(diff.map(&:class)).to eq [ Dyph3::Action::Delete, Dyph3::Action::Add, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::Delete, Dyph3::Action::Add, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::Delete, Dyph3::Action::Delete, Dyph3::Action::Add, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::NoChange, Dyph3::Action::Delete, Dyph3::Action::Add ]
        end
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.20263997679141282, 'fraction_numerical': 0.05483028720626632, 'mean_word_length': 3.9285203716940673, 'pattern_counts': {'":': 41, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3824177', 'n_tokens_mistral': 2695, 'n_tokens_neox': 2614, 'n_words': 509}