<commit_before>#encoding: utf-8
require 'find'

class ToolsController < ApplicationController
  def cluster_symetry
    symetry_for(/sgbd-/)
    @title = "serveurs SGBD"
    render 'servers_symetry'
  end

  def acai_symetry
    symetry_for(/^vm-(preprod|prod|ecole|web)/)
    @title = "serveurs ACAI"
    render 'servers_symetry'
  end

  def nagios_comparison
    path = Rails.root.join("data/nagios")
    servers_nagios = %x(grep host_name= #{path}/status.dat|cut -d"=" -f 2|sort -u).split(/\n/)
    # service checks that can't be renamed easily in nagios
    servers_nagios.reject!{|h| h.match(/^(VIPS?\d*_|GW_|URL_|sgbd-prod[abc]$|sgbd-prod[abc]-[a-z])/) }
    servers_nagios.each{|h| h.gsub!(/^(MIN_|RH_|RITAC_|AM_|DEV_)/,"") }
    servers_cartocs = Server.all

    @unknown_in_nagios = servers_cartocs.map(&:name) - servers_nagios
    #RH project we're not responsible of
    @unknown_in_nagios.reject!{ |name| name.starts_with?("vm-hra") || name.starts_with?("vm-hrw") }
    #Windows servers we're not responsible of
    @unknown_in_nagios.reject!{ |name| name.match(/millos|ritac-|-nt-|-ac-|^dns-[01]$/) }
    @unknown_in_cartocs = servers_nagios - servers_cartocs.map(&:name)

    tomcats_jason = Tomcat.all.map{|t| t[:tomcat]+"@"+t[:server]}.select{|t| t.match(/\S@\S/)}
    tomcats_nagios = []; prev=""
    datafile = "#{Rails.root}/data/nagios/status.dat"
    File.open(datafile).each_line do |line|
      tomcats_nagios << line.split("PROCESS_").last.chomp + "@" + prev.split("=").last.chomp if "PROCESS_TC".in?(line)
      prev=line
    end if File.exists?(datafile)
    @tomcat_unknown_in_nagios = tomcats_jason - tomcats_nagios
    @tomcat_unknown_in_nagios.reject!{|t| t.match(/vm-preprod/) }
    @nb_tomcats = tomcats_jason.reject{|t| t.match(/vm-preprod/) }.count
  end

  def reverse_proxies_comparison
    path = Rails.root.join("data/rp")
    @vhosts_lines = %x(grep -rHE '^\s*ServerName' #{path} |grep -v sites-available|sort -u 2>/dev/null).split(/\n+/)
    vhosts = @vhosts_lines.map{|line| line.scan(/ServerName\s+(\S+)/).first.first}.flatten.compact.uniq

    app_urls = ApplicationUrl.all.map(&:url).map{|u| u.strip.gsub(%r{\S+://},"").gsub(%r{/.*},"")}

    @unknown_in_cartocs = vhosts - app_urls
  end

  def vmware_comparison
    vms = []
    datafile = Rails.root.join("data/vmware/virtual_servers.txt")
    vms = File.readlines(datafile).map(&:chomp!) if File.exists?(datafile)
    vms_in_cartocs = Server.where(:virtual => true)
    physical_in_cartocs = Server.where(:virtual => false)

    @unknown_in_cartocs = vms - vms_in_cartocs.map(&:name) - physical_in_cartocs.map(&:name)
    @unknown_in_vcenter = vms_in_cartocs.map(&:name) - vms
    #RH project we're not responsible of
    @unknown_in_vcenter.reject!{ |name| name.starts_with?("vm-hra") || name.starts_with?("vm-hrw") }
    #Windows servers we're not responsible of
    @unknown_in_vcenter.reject!{ |name| name.match(/millos|ritac-|-nt-|-ac-|^dns-[01]$/) }
    @not_vms_in_cartocs = vms - vms_in_cartocs.map(&:name) - @unknown_in_cartocs
  end

  protected
  def symetry_for(mask)
    path = Rails.root.join("data/symetry")
    @clusters = {}
    Dir.glob("#{path}/*").map{|c| File.basename(c)}.sort.each do |cluster|
      next unless cluster.match(mask)
      @clusters[cluster] = { :path => "#{path}/#{cluster}",
                            :nodes => Dir.glob("#{path}/#{cluster}/*").map{|c| File.basename(c)}.sort }
      files = []
      lists = []
      Find.find(@clusters[cluster][:path]).each do |file|
        if file.ends_with?(".list")
          lists << file.sub(@clusters[cluster][:path],"").sub(%r{^/[^/]+},"")
        elsif File.file?(file)
          files << file.sub(@clusters[cluster][:path],"").sub(%r{^/[^/]+},"")
        end
      end
      @clusters[cluster][:files] = files.sort.uniq
      @clusters[cluster][:lists] = lists.reject{|f| f.ends_with?("packages.list")}.sort.uniq
    end
    @cluster_names = @clusters.keys
    #limit clusters
    @clusters.reject! do |cluster,hsh|
      cluster != params[:id]
    end unless params[:id] == "all"
  end
end<commit_msg>tools/servers_symetry optimization: don't retrieve data from clusters if we just want their names<commit_after>#encoding: utf-8
require 'find'

class ToolsController < ApplicationController
  def cluster_symetry
    symetry_for(/sgbd-/)
    @title = "serveurs SGBD"
    render 'servers_symetry'
  end

  def acai_symetry
    symetry_for(/^vm-(preprod|prod|ecole|web)/)
    @title = "serveurs ACAI"
    render 'servers_symetry'
  end

  def nagios_comparison
    path = Rails.root.join("data/nagios")
    servers_nagios = %x(grep host_name= #{path}/status.dat|cut -d"=" -f 2|sort -u).split(/\n/)
    # service checks that can't be renamed easily in nagios
    servers_nagios.reject!{|h| h.match(/^(VIPS?\d*_|GW_|URL_|sgbd-prod[abc]$|sgbd-prod[abc]-[a-z])/) }
    servers_nagios.each{|h| h.gsub!(/^(MIN_|RH_|RITAC_|AM_|DEV_)/,"") }
    servers_cartocs = Server.all

    @unknown_in_nagios = servers_cartocs.map(&:name) - servers_nagios
    #RH project we're not responsible of
    @unknown_in_nagios.reject!{ |name| name.starts_with?("vm-hra") || name.starts_with?("vm-hrw") }
    #Windows servers we're not responsible of
    @unknown_in_nagios.reject!{ |name| name.match(/millos|ritac-|-nt-|-ac-|^dns-[01]$/) }
    @unknown_in_cartocs = servers_nagios - servers_cartocs.map(&:name)

    tomcats_jason = Tomcat.all.map{|t| t[:tomcat]+"@"+t[:server]}.select{|t| t.match(/\S@\S/)}
    tomcats_nagios = []; prev=""
    datafile = "#{Rails.root}/data/nagios/status.dat"
    File.open(datafile).each_line do |line|
      tomcats_nagios << line.split("PROCESS_").last.chomp + "@" + prev.split("=").last.chomp if "PROCESS_TC".in?(line)
      prev=line
    end if File.exists?(datafile)
    @tomcat_unknown_in_nagios = tomcats_jason - tomcats_nagios
    @tomcat_unknown_in_nagios.reject!{|t| t.match(/vm-preprod/) }
    @nb_tomcats = tomcats_jason.reject{|t| t.match(/vm-preprod/) }.count
  end

  def reverse_proxies_comparison
    path = Rails.root.join("data/rp")
    @vhosts_lines = %x(grep -rHE '^\s*ServerName' #{path} |grep -v sites-available|sort -u 2>/dev/null).split(/\n+/)
    vhosts = @vhosts_lines.map{|line| line.scan(/ServerName\s+(\S+)/).first.first}.flatten.compact.uniq

    app_urls = ApplicationUrl.all.map(&:url).map{|u| u.strip.gsub(%r{\S+://},"").gsub(%r{/.*},"")}

    @unknown_in_cartocs = vhosts - app_urls
  end

  def vmware_comparison
    vms = []
    datafile = Rails.root.join("data/vmware/virtual_servers.txt")
    vms = File.readlines(datafile).map(&:chomp!) if File.exists?(datafile)
    vms_in_cartocs = Server.where(:virtual => true)
    physical_in_cartocs = Server.where(:virtual => false)

    @unknown_in_cartocs = vms - vms_in_cartocs.map(&:name) - physical_in_cartocs.map(&:name)
    @unknown_in_vcenter = vms_in_cartocs.map(&:name) - vms
    #RH project we're not responsible of
    @unknown_in_vcenter.reject!{ |name| name.starts_with?("vm-hra") || name.starts_with?("vm-hrw") }
    #Windows servers we're not responsible of
    @unknown_in_vcenter.reject!{ |name| name.match(/millos|ritac-|-nt-|-ac-|^dns-[01]$/) }
    @not_vms_in_cartocs = vms - vms_in_cartocs.map(&:name) - @unknown_in_cartocs
  end

  protected
  def symetry_for(mask)
    path = Rails.root.join("data/symetry")
    @clusters = {}
    @cluster_names = []
    Dir.glob("#{path}/*").map{|c| File.basename(c)}.sort.each do |cluster|
      next unless cluster.match(mask)
      @cluster_names << cluster
      next unless cluster == params[:id] || params[:id] == "all"
      @clusters[cluster] = { :path => "#{path}/#{cluster}",
                            :nodes => Dir.glob("#{path}/#{cluster}/*").map{|c| File.basename(c)}.sort }
      files = []
      lists = []
      Find.find(@clusters[cluster][:path]).each do |file|
        if file.ends_with?(".list")
          lists << file.sub(@clusters[cluster][:path],"").sub(%r{^/[^/]+},"")
        elsif File.file?(file)
          files << file.sub(@clusters[cluster][:path],"").sub(%r{^/[^/]+},"")
        end
      end
      @clusters[cluster][:files] = files.sort.uniq
      @clusters[cluster][:lists] = lists.reject{|f| f.ends_with?("packages.list")}.sort.uniq
    end
  end
end
{'fraction_non_alphanumeric': 0.18671694441065564, 'fraction_numerical': 0.0017029558447877388, 'mean_word_length': 3.6636415201361316, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2629597', 'n_tokens_mistral': 3258, 'n_tokens_neox': 3133, 'n_words': 471}