<commit_before><%
require 'find'
path = Rails.root.join("data/symetry")
clusters = {}
Dir.glob("#{path}/*").map{|c| File.basename(c)}.sort.each do |cluster|
  clusters[cluster] = {:path => "#{path}/#{cluster}",
                       :nodes => Dir.glob("#{path}/#{cluster}/*").map{|c| File.basename(c)}.sort}
  files = []
  lists = []
  Find.find(clusters[cluster][:path]).each do |file|
    if file.ends_with?(".list")
      lists << file.sub(clusters[cluster][:path],"").sub(%r{^/[^/]+},"")
    elsif File.file?(file)
      files << file.sub(clusters[cluster][:path],"").sub(%r{^/[^/]+},"")
    end
  end
  clusters[cluster][:files] = files.sort.uniq
  clusters[cluster][:lists] = lists.sort.uniq
end
%>

<% title "Outils - Symétrie des clusters" %>

<div id="tools" class="grid_12 items_list">

<%= render "submenu" %>

<%= action_links do %>
  <ul class="select-buttons">
  <% clusters.keys.each do |cluster| %>
    <%= content_tag :li, link_to(cluster, tool_path(:cluster_symetry, :id => cluster)), :class => (params[:id] == cluster ? "current" : "") %>
  <% end %>
  </ul>
<% end %>

<%
#limit clusters
clusters.reject!{|cluster,hsh| cluster != params[:id] } unless params[:id] == "all"
%>

<% clusters.each do |cluster,hsh| %>
<h2><%= cluster.upcase %></h2>
<table class="list symetry">
  <tr>
    <th style="width:10%;"> </th>
    <% hsh[:nodes].each do |node| %>
      <th style="width:<%= 90 / hsh[:nodes].size %>%;"><%= node %></th>
    <% end %>
  </tr>

  <% # FILES %>
  <% hsh[:files].each do |file| %>
    <% content = hsh[:nodes].map{|n| File.readlines("#{hsh[:path]}/#{n}/#{file}").reject{|l|l.starts_with?("#") || l.blank?}.join.chomp } %>
    <% uniq = content.uniq %>
    <% if uniq.length == 1 %>
    <tr class="identical">
      <td><%= file %></td><td colspan="<%= content.length %>"><code><%= content.shift.gsub("\n","<br/>").html_safe %></code></td>
    </tr>
    <% else %>
    <tr class="different">
      <td><%= file %></td><% content.length.times do %><td><code><%= content.shift.gsub("\n","<br/>").html_safe %></code></td><% end %>
    </tr>
    <% end %>
  <% end %>

  <% # PACKAGES WITHOUT VERSION %>
  <% packages_map = hsh[:nodes].map do |n|
       File.readlines("#{hsh[:path]}/#{n}/packages.list").map{|l| l.sub(/-[0-9].*/, "")}.join("\n")
     end %>
  <% uniq = packages_map.uniq %>
  <% if uniq.length == 1 %>
  <tr class="identical">
    <td>Packages (hors version)</td><td colspan="<%= packages_map.length %>"><%= packages_map.first.count("\n")+1 %> packages</td>
  </tr>
  <% else %>
  <tr class="different">
    <td>Packages (hors version)</td>
    <% packages_map.each_with_index do |packages,index| %>
      <td>
        <%= packages.lines.count %> packages.
        <% if index == 0 %>
          (référence)
        <% else %>
          <% diff = Diffy::Diff.new(packages_map.first, packages, :diff => %w(-B -b -w -E --unified=0)).to_s(:html_simple) %>
          <% diff.gsub!(%(<li class="ins"><ins></ins></li>),"") %>
          <% diff.gsub!(%(<li class="del"><del></del></li>),"") %>
          <%= diff.scan("<ins>").count + diff.scan("<del>").count %> différences avec <%= hsh[:nodes].first %>:<br />
          <%= diff.html_safe %>
        <% end %>
      </td>
    <% end %>
  </tr>
  <% end %>



  <% # PACKAGES WITH VERSION %>
  <% packages_map = hsh[:nodes].map do |n|
       File.read("#{hsh[:path]}/#{n}/packages.list")
     end %>
  <% uniq = packages_map.uniq %>
  <% if uniq.length == 1 %>
  <tr class="identical">
    <td>Packages (avec version)</td><td colspan="<%= packages_map.length %>"><%= packages_map.first.count("\n")+1 %> packages</td>
  </tr>
  <% else %>
  <tr class="different">
    <td>Packages (avec version)</td>
    <% packages_map.each_with_index do |packages,index| %>
      <td>
        <%= packages.lines.count %> packages.
        <% if index == 0 %>
          (référence)
        <% else %>
          <% diff = Diffy::Diff.new(packages_map.first, packages, :diff => %w(-B -b -w -E --unified=0)).to_s(:html_simple) %>
          <% diff.gsub!(%(<li class="ins"><ins></ins></li>),"") %>
          <% diff.gsub!(%(<li class="del"><del></del></li>),"") %>
          <%= diff.scan("<ins>").count + diff.scan("<del>").count %> différences avec <%= hsh[:nodes].first %>:<br />
          <%= diff.html_safe %>
        <% end %>
      </td>
    <% end %>
  </tr>
  <% end %>

<% # FILES HIERARCHY (LISTS) %>
<% hsh[:lists].each do |file_list| %>
  <% lists_map = hsh[:nodes].map do |n|
       File.read("#{hsh[:path]}/#{n}/#{file_list}")
     end %>
  <% hierarchy = file_list.gsub(%r{^/list},"").gsub(/\.list$/,"").gsub("_","/") %>
  <% uniq = lists_map.uniq %>
  <% if uniq.length == 1 %>
  <tr class="identical">
    <td>Fichiers <%= hierarchy %></td><td colspan="<%= lists_map.length %>"><%= lists_map.first.lines.count %> fichiers identiques (fichier,permissions,md5sum)</td>
  </tr><commit_msg>Fixed 500 error when symetry files do not exist<commit_after><%
require 'find'
path = Rails.root.join("data/symetry")
clusters = {}
Dir.glob("#{path}/*").map{|c| File.basename(c)}.sort.each do |cluster|
  clusters[cluster] = {:path => "#{path}/#{cluster}",
                       :nodes => Dir.glob("#{path}/#{cluster}/*").map{|c| File.basename(c)}.sort}
  files = []
  lists = []
  Find.find(clusters[cluster][:path]).each do |file|
    if file.ends_with?(".list")
      lists << file.sub(clusters[cluster][:path],"").sub(%r{^/[^/]+},"")
    elsif File.file?(file)
      files << file.sub(clusters[cluster][:path],"").sub(%r{^/[^/]+},"")
    end
  end
  clusters[cluster][:files] = files.sort.uniq
  clusters[cluster][:lists] = lists.sort.uniq
end
%>

<% title "Outils - Symétrie des clusters" %>

<div id="tools" class="grid_12 items_list">

<%= render "submenu" %>

<%= action_links do %>
  <ul class="select-buttons">
  <% clusters.keys.each do |cluster| %>
    <%= content_tag :li, link_to(cluster, tool_path(:cluster_symetry, :id => cluster)), :class => (params[:id] == cluster ? "current" : "") %>
  <% end %>
  </ul>
<% end %>

<%
#limit clusters
clusters.reject!{|cluster,hsh| cluster != params[:id] } unless params[:id] == "all"
%>

<% clusters.each do |cluster,hsh| %>
<h2><%= cluster.upcase %></h2>
<table class="list symetry">
  <tr>
    <th style="width:10%;"> </th>
    <% hsh[:nodes].each do |node| %>
      <th style="width:<%= 90 / hsh[:nodes].size %>%;"><%= node %></th>
    <% end %>
  </tr>

  <% # FILES %>
  <% hsh[:files].each do |file| %>
    <% content = hsh[:nodes].map do |n|
         file_path = "#{hsh[:path]}/#{n}/#{file}"
         File.exists?(file_path) ? File.readlines(file_path).reject{|l|l.starts_with?("#") || l.blank?}.join.chomp : []
       end %>
    <% uniq = content.uniq %>
    <% if uniq.length == 1 %>
    <tr class="identical">
      <td><%= file %></td><td colspan="<%= content.length %>"><code><%= content.shift.gsub("\n","<br/>").html_safe %></code></td>
    </tr>
    <% else %>
    <tr class="different">
      <td><%= file %></td><% content.length.times do %><td><code><%= content.shift.gsub("\n","<br/>").html_safe %></code></td><% end %>
    </tr>
    <% end %>
  <% end %>

  <% # PACKAGES WITHOUT VERSION %>
  <% packages_map = hsh[:nodes].map do |n|
       pkg_file = "#{hsh[:path]}/#{n}/packages.list"
       File.exists?(pkg_file) ? File.readlines(pkg_file).map{|l| l.sub(/-[0-9].*/, "")}.join("\n") : []
     end %>
  <% uniq = packages_map.uniq %>
  <% if uniq.length == 1 %>
  <tr class="identical">
    <td>Packages (hors version)</td><td colspan="<%= packages_map.length %>"><%= packages_map.first.count("\n")+1 %> packages</td>
  </tr>
  <% else %>
  <tr class="different">
    <td>Packages (hors version)</td>
    <% packages_map.each_with_index do |packages,index| %>
      <td>
        <%= packages.lines.count %> packages.
        <% if index == 0 %>
          (référence)
        <% else %>
          <% diff = Diffy::Diff.new(packages_map.first, packages, :diff => %w(-B -b -w -E --unified=0)).to_s(:html_simple) %>
          <% diff.gsub!(%(<li class="ins"><ins></ins></li>),"") %>
          <% diff.gsub!(%(<li class="del"><del></del></li>),"") %>
          <%= diff.scan("<ins>").count + diff.scan("<del>").count %> différences avec <%= hsh[:nodes].first %>:<br />
          <%= diff.html_safe %>
        <% end %>
      </td>
    <% end %>
  </tr>
  <% end %>



  <% # PACKAGES WITH VERSION %>
  <% packages_map = hsh[:nodes].map do |n|
       pkg_file = "#{hsh[:path]}/#{n}/packages.list"
       File.exists?(pkg_file) ? File.readlines(pkg_file).map{|l| l.sub(/-[0-9].*/, "")}.join("\n") : []
     end %>
  <% uniq = packages_map.uniq %>
  <% if uniq.length == 1 %>
  <tr class="identical">
    <td>Packages (avec version)</td><td colspan="<%= packages_map.length %>"><%= packages_map.first.count("\n")+1 %> packages</td>
  </tr>
  <% else %>
  <tr class="different">
    <td>Packages (avec version)</td>
    <% packages_map.each_with_index do |packages,index| %>
      <td>
        <%= packages.lines.count %> packages.
        <% if index == 0 %>
          (référence)
        <% else %>
          <% diff = Diffy::Diff.new(packages_map.first, packages, :diff => %w(-B -b -w -E --unified=0)).to_s(:html_simple) %>
          <% diff.gsub!(%(<li class="ins"><ins></ins></li>),"") %>
          <% diff.gsub!(%(<li class="del"><del></del></li>),"") %>
          <%= diff.scan("<ins>").count + diff.scan("<del>").count %> différences avec <%= hsh[:nodes].first %>:<br />
          <%= diff.html_safe %>
        <% end %>
      </td>
    <% end %>
  </tr>
  <% end %>

<% # FILES HIERARCHY (LISTS) %>
<% hsh[:lists].each do |file_list| %>
  <% lists_map = hsh[:nodes].map do |n|
       file_path = "#{hsh[:path]}/#{n}/#{file_list}"
       File.exists?(file_path) ? File.read(file_path) : []
     end %>
  <% hierarchy = file_list.gsub(%r{^/list},"").gsub(/\.list$/,"").gsub("_","/") %>
  <% uniq = lists_map.uniq %>
  <% if uniq.length == 1 %>
  <tr class="identical">
    <td>Fichiers <%= hierarchy %></td><td colspan="<%= lists_map.length %>"><%= lists_map.first.lines.count %> fichiers identiques (fichier,permissions,md5sum)</td>
  </tr>
{'fraction_non_alphanumeric': 0.26347602061038444, 'fraction_numerical': 0.004657154181529925, 'mean_word_length': 3.109527687296417, 'pattern_counts': {'":': 0, '<': 333, '<?xml version=': 0, '>': 337, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1774557', 'n_tokens_mistral': 4075, 'n_tokens_neox': 3718, 'n_words': 630}