<commit_before><%
require 'find'
path = Rails.root.join("data/symetry")
clusters = {}
Dir.glob("#{path}/*").map{|c| File.basename(c)}.sort.each do |cluster|
  clusters[cluster] = {:path => "#{path}/#{cluster}",
                       :nodes => Dir.glob("#{path}/#{cluster}/*").map{|c| File.basename(c)}.sort}
  files = []
  lists = []
  Find.find(clusters[cluster][:path]).each do |file|
    if file.ends_with?(".list")
      lists << file.sub(clusters[cluster][:path],"").sub(%r{^/[^/]+},"")
    elsif File.file?(file)
      files << file.sub(clusters[cluster][:path],"").sub(%r{^/[^/]+},"")
    end
  end
  clusters[cluster][:files] = files.sort.uniq
  clusters[cluster][:lists] = lists.reject{|f| f.ends_with?("packages.list")}.sort.uniq
end
%>

<% title "Outils - Symétrie des clusters" %>

<div id="cluster_symetry" class="grid_12 items_list">

<%= action_links do %>
  <ul class="select-buttons">
  <% clusters.keys.each do |cluster| %>
    <%= content_tag :li, link_to(cluster, tool_path(:cluster_symetry, :id => cluster)), :class => (params[:id] == cluster ? "current" : "") %>
  <% end %>
  </ul>
<% end %>

<%
#limit clusters
clusters.reject!{|cluster,hsh| cluster != params[:id] } unless params[:id] == "all"
%>

<% clusters.each do |cluster,hsh| %>

<% # FILES %>
<% hsh[:files].each do |file| %>
  <% blank = 0 %>
  <% content = hsh[:nodes].map do |n|
       file_path = "#{hsh[:path]}/#{n}/#{file}"
       lines = File.exists?(file_path) ? File.readlines(file_path).reject{|l|l.starts_with?("#") || l.blank?}.join.chomp : ""
       blank += 1 if lines.blank?
       lines 
     end %>
  <% next if blank == content.size %>
  <% uniq = content.uniq %>
  <% next if content == [[], []] %>
  <% next if content == [[""], [""]] %>
  <%= symetry_table_for hsh[:nodes], :status => (uniq.length == 1), :title => file do %>
    <tr>
    <% if uniq.length == 1 %>
      <td colspan="<%= content.length %>"><code><%= content.shift.gsub("\n","<br/>").html_safe %></code></td>
    <% else %>
      <% content.length.times do %><td><code><%= content.shift.gsub("\n","<br/>").html_safe %></code></td><% end %>
    <% end %>
    </tr>
  <% end %>
<% end %>

<% # PACKAGES WITHOUT VERSION
packages_map = hsh[:nodes].map do |n|
  pkg_file = "#{hsh[:path]}/#{n}/packages.list"
  File.exists?(pkg_file) ? File.readlines(pkg_file).map{|l| l.chomp.sub(/-[0-9].*/, "")}.join("\n") : ""
end
uniq = packages_map.uniq %>
<%= symetry_table_for hsh[:nodes], :status => (uniq.length == 1), :title => "Packages (hors version)" do %>
  <% if uniq.length == 1 %>
  <tr>
    <td colspan="<%= packages_map.length %>"><%= packages_map.first.count("\n")+1 %> packages</td>
  </tr>
  <% else %>
  <tr>
    <% packages_map.each_with_index do |packages,index| %>
      <td>
        <%= packages.lines.count %> packages.
        <% if index == 0 %>
          (référence)
        <% else %>
          <% diff = Diffy::Diff.new(packages_map.first, packages, :diff => %w(-B -b -w -E --unified=0)).to_s(:html_simple) %>
          <% diff.gsub!(%(<li class="ins"><ins></ins></li>),"") %>
          <% diff.gsub!(%(<li class="del"><del></del></li>),"") %>
          <%= diff.scan("<ins>").count + diff.scan("<del>").count %> différences avec <%= hsh[:nodes].first %>:<br />
          <%= diff.html_safe %>
        <% end %>
      </td>
    <% end %>
  </tr>
  <% end %>
<% end %>

<% # PACKAGES WITH VERSION
packages_map = hsh[:nodes].map do |n|
  pkg_file = "#{hsh[:path]}/#{n}/packages.list"
  File.exists?(pkg_file) ? File.read(pkg_file) : ""
end
uniq = packages_map.uniq %>
<%= symetry_table_for hsh[:nodes], :status => (uniq.length == 1), :title => "Packages (avec version)" do %>
  <% if uniq.length == 1 %>
  <tr class="identical">
    <td colspan="<%= packages_map.length %>"><%= packages_map.first.count("\n")+1 %> packages</td>
  </tr>
  <% else %>
  <tr class="different">
    <% packages_map.each_with_index do |packages,index| %>
      <td>
        <%= packages.lines.count %> packages.
        <% if index == 0 %>
          (référence)
        <% else %>
          <% diff = Diffy::Diff.new(packages_map.first, packages, :diff => %w(-B -b -w -E --unified=0)).to_s(:html_simple) %>
          <% diff.gsub!(%(<li class="ins"><ins></ins></li>),"") %>
          <% diff.gsub!(%(<li class="del"><del></del></li>),"") %>
          <%= diff.scan("<ins>").count + diff.scan("<del>").count %> différences avec <%= hsh[:nodes].first %>:<br />
          <%= diff.html_safe %>
        <% end %>
      </td>
    <% end %>
  </tr>
  <% end %>
<% end %>

<% # FILES HIERARCHY (LISTS)
hsh[:lists].each do |file_list|
lists_map = hsh[:nodes].map do |n|
  file_path = "#{hsh[:path]}/#{n}/#{file_list}"
  File.exists?(file_path) ? File.read(file_path) : ""
end
hierarchy = file_list.gsub(%r{^/list},"").gsub(/\.list$/,"").gsub("_","/")
uniq = lists_map.uniq %>
<%= symetry_table_for hsh[:nodes], :status => (uniq.length == 1), :title => "Fichiers #{hierarchy}" do %>
  <% if uniq.length == 1 %>
  <tr>
    <td colspan="<%= lists_map.length %>"><%= lists_map.first.lines.count %> fichiers identiques (fichier,permissions,md5sum)</td>
  </tr>
  <% else %>
  <tr>
    <% lists_map.each_with_index do |files,index| %>
      <td>
        <%= files.lines.count %> fichiers.
        <% if index == 0 %>
          (référence)
        <% else %>
          <% diff = Diffy::Diff.new(lists_map.first, files, :diff => %w(-B -b -w -E --unified=0)).to_s(:html_simple) %>
          <% diff.gsub!(%(<li class="ins"><ins></ins></li>),"") %>
          <% diff.gsub!(%(<li class="del"><del></del></li>),"") %>
          <%= diff.scan("<ins>").count + diff.scan("<del>").count %> différences avec <%= hsh[:nodes].first %>:<br />
          <%= diff.html_safe %>
        <% end %>
      </td>
    <% end %>
  </tr>
  <% end %>
<% end %>
<% end %>

<% end %>
 
</div>
<commit_msg>Refined the way we evalute diff between files in tools/cluster_symetry<commit_after><%
require 'find'
path = Rails.root.join("data/symetry")
clusters = {}
Dir.glob("#{path}/*").map{|c| File.basename(c)}.sort.each do |cluster|
  clusters[cluster] = {:path => "#{path}/#{cluster}",
                       :nodes => Dir.glob("#{path}/#{cluster}/*").map{|c| File.basename(c)}.sort}
  files = []
  lists = []
  Find.find(clusters[cluster][:path]).each do |file|
    if file.ends_with?(".list")
      lists << file.sub(clusters[cluster][:path],"").sub(%r{^/[^/]+},"")
    elsif File.file?(file)
      files << file.sub(clusters[cluster][:path],"").sub(%r{^/[^/]+},"")
    end
  end
  clusters[cluster][:files] = files.sort.uniq
  clusters[cluster][:lists] = lists.reject{|f| f.ends_with?("packages.list")}.sort.uniq
end
%>

<% title "Outils - Symétrie des clusters" %>

<div id="cluster_symetry" class="grid_12 items_list">

<%= action_links do %>
  <ul class="select-buttons">
  <% clusters.keys.each do |cluster| %>
    <%= content_tag :li, link_to(cluster, tool_path(:cluster_symetry, :id => cluster)), :class => (params[:id] == cluster ? "current" : "") %>
  <% end %>
  </ul>
<% end %>

<%
#limit clusters
clusters.reject!{|cluster,hsh| cluster != params[:id] } unless params[:id] == "all"
%>

<% clusters.each do |cluster,hsh| %>

<% # FILES %>
<% hsh[:files].each do |file| %>
  <% blank = 0 %>
  <% content = hsh[:nodes].map do |n|
       file_path = "#{hsh[:path]}/#{n}/#{file}"
       lines = File.exists?(file_path) ? File.readlines(file_path).reject{|l|l.starts_with?("#") || l.blank?}.join.chomp : ""
       blank += 1 if lines.blank?
       lines 
     end
     next if blank == content.size
     next if content == [[], []]
     next if content == [[""], [""]]
     
     status = content.inject(true) do |memo,file|
       equality = Diffy::Diff.new(content.first, file, :diff => %w(-B -b -w -E --unified=0)).inject(true) do |memo,chunk|
         memo && !chunk.match(/^-|^\+/)
       end
       memo && equality
     end
  %>
  <%= symetry_table_for hsh[:nodes], :status => status, :title => file do %>
    <tr>
    <% if status %>
      <td colspan="<%= content.length %>"><code><%= content.shift.gsub("\n","<br/>").html_safe %></code></td>
    <% else %>
      <% content.length.times do %><td><code><%= content.shift.gsub("\n","<br/>").html_safe %></code></td><% end %>
    <% end %>
    </tr>
  <% end %>
<% end %>

<% # PACKAGES WITHOUT VERSION
packages_map = hsh[:nodes].map do |n|
  pkg_file = "#{hsh[:path]}/#{n}/packages.list"
  File.exists?(pkg_file) ? File.readlines(pkg_file).map{|l| l.chomp.sub(/-[0-9].*/, "")}.join("\n") : ""
end
uniq = packages_map.uniq %>
<%= symetry_table_for hsh[:nodes], :status => (uniq.length == 1), :title => "Packages (hors version)" do %>
  <% if uniq.length == 1 %>
  <tr>
    <td colspan="<%= packages_map.length %>"><%= packages_map.first.count("\n")+1 %> packages</td>
  </tr>
  <% else %>
  <tr>
    <% packages_map.each_with_index do |packages,index| %>
      <td>
        <%= packages.lines.count %> packages.
        <% if index == 0 %>
          (référence)
        <% else %>
          <% diff = Diffy::Diff.new(packages_map.first, packages, :diff => %w(-B -b -w -E --unified=0)).to_s(:html_simple) %>
          <% diff.gsub!(%(<li class="ins"><ins></ins></li>),"") %>
          <% diff.gsub!(%(<li class="del"><del></del></li>),"") %>
          <%= diff.scan("<ins>").count + diff.scan("<del>").count %> différences avec <%= hsh[:nodes].first %>:<br />
          <%= diff.html_safe %>
        <% end %>
      </td>
    <% end %>
  </tr>
  <% end %>
<% end %>

<% # PACKAGES WITH VERSION
packages_map = hsh[:nodes].map do |n|
  pkg_file = "#{hsh[:path]}/#{n}/packages.list"
  File.exists?(pkg_file) ? File.read(pkg_file) : ""
end
uniq = packages_map.uniq %>
<%= symetry_table_for hsh[:nodes], :status => (uniq.length == 1), :title => "Packages (avec version)" do %>
  <% if uniq.length == 1 %>
  <tr class="identical">
    <td colspan="<%= packages_map.length %>"><%= packages_map.first.count("\n")+1 %> packages</td>
  </tr>
  <% else %>
  <tr class="different">
    <% packages_map.each_with_index do |packages,index| %>
      <td>
        <%= packages.lines.count %> packages.
        <% if index == 0 %>
          (référence)
        <% else %>
          <% diff = Diffy::Diff.new(packages_map.first, packages, :diff => %w(-B -b -w -E --unified=0)).to_s(:html_simple) %>
          <% diff.gsub!(%(<li class="ins"><ins></ins></li>),"") %>
          <% diff.gsub!(%(<li class="del"><del></del></li>),"") %>
          <%= diff.scan("<ins>").count + diff.scan("<del>").count %> différences avec <%= hsh[:nodes].first %>:<br />
          <%= diff.html_safe %>
        <% end %>
      </td>
    <% end %>
  </tr>
  <% end %>
<% end %>

<% # FILES HIERARCHY (LISTS)
hsh[:lists].each do |file_list|
lists_map = hsh[:nodes].map do |n|
  file_path = "#{hsh[:path]}/#{n}/#{file_list}"
  File.exists?(file_path) ? File.read(file_path) : ""
end
hierarchy = file_list.gsub(%r{^/list},"").gsub(/\.list$/,"").gsub("_","/")
uniq = lists_map.uniq %>
<%= symetry_table_for hsh[:nodes], :status => (uniq.length == 1), :title => "Fichiers #{hierarchy}" do %>
  <% if uniq.length == 1 %>
  <tr>
    <td colspan="<%= lists_map.length %>"><%= lists_map.first.lines.count %> fichiers identiques (fichier,permissions,md5sum)</td>
  </tr>
  <% else %>
  <tr>
    <% lists_map.each_with_index do |files,index| %>
      <td>
        <%= files.lines.count %> fichiers.
        <% if index == 0 %>
          (référence)
        <% else %>
          <% diff = Diffy::Diff.new(lists_map.first, files, :diff => %w(-B -b -w -E --unified=0)).to_s(:html_simple) %>
          <% diff.gsub!(%(<li class="ins"><ins></ins></li>),"") %>
          <% diff.gsub!(%(<li class="del"><del></del></li>),"") %>
          <%= diff.scan("<ins>").count + diff.scan("<del>").count %> différences avec <%= hsh[:nodes].first %>:<br />
          <%= diff.html_safe %>
        <% end %>
      </td>
    <% end %>
  </tr>
  <% end %>
<% end %>
<% end %>

<% end %>
 
</div>

{'fraction_non_alphanumeric': 0.2555443548387097, 'fraction_numerical': 0.003780241935483871, 'mean_word_length': 3.0534559073884915, 'pattern_counts': {'":': 0, '<': 333, '<?xml version=': 0, '>': 356, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4878692', 'n_tokens_mistral': 4788, 'n_tokens_neox': 4395, 'n_words': 770}