<commit_before>desc "Imports tina policies from data/tina/*.tina.dump config dumps"
namespace :import do
  task :tina => :environment do
    sections = {}
    #collect sections
    Dir.glob("data/tina/*.tina.dump").each do |file|
      catalog = file.gsub(/.*catalog_/,"").gsub(".tina.dump","")
      hostname = nil
      attributes = {}
      File.read(file).each_line do |line|
        if line.match(/^(?:SYSTEME|- Syst\*me)\s+:\s+(\S+)(?:\s+\[(\S+)\])?/)
          sections[hostname] = attributes if hostname
          hostname = $1
          attributes = {:catalog => catalog, :type => $2, :files => []}
        elsif line.match(/Version logicielle : (.*)/)
          attributes[:client_version] = $1
        elsif line.match(%r{^\s{2,}(/\S*)})
          attributes[:files] << $1
        end
      end
      sections[hostname] = attributes if hostname
    end
    #save data in the db
    sections.each do |servername, attributes|
      server = Server.find_or_generate(servername)
      attributes[:files].each do |fs|
        job = BackupJob.find_or_create_by_server_id_and_hierarchy(server.id, fs)
        job.client_type = "TiNa"
        job.client_version = attributes[:client_version]
        job.catalog = attributes[:catalog]
        job.save if job.changed?
      end
    end
  end
end
<commit_msg>Improved import:tina rake task<commit_after>desc "Imports tina policies from data/tina/*.tina.dump config dumps"
namespace :import do
  task :tina => :environment do
    sections = {}
    #collect sections
    Dir.glob("data/tina/*.tina.dump").each do |file|
      catalog = file.gsub(/.*catalog_/,"").gsub(".tina.dump","")
      hostname = nil
      attributes = {}
      File.read(file).each_line do |line|
        if line.match(/^(?:SYSTEME|- Syst\*me)\s+:\s+(\S+)(?:\s+\[(\S+)\])?/)
          if hostname
            sections[hostname] ||= []
            sections[hostname] << attributes
          end
          hostname = $1
          attributes = {:catalog => catalog, :type => $2, :files => []}
        elsif line.match(/Version logicielle : (.*)/)
          attributes[:client_version] = $1
        elsif line.match(%r{^\s{2,}(/\S*)})
          attributes[:files] << $1
        end
      end
      if hostname
        sections[hostname] ||= []
        sections[hostname] << attributes
      end
    end
    #save data in the db
    sections.each do |servername, jobs|
      server = Server.find_or_generate(servername)
      jobs.each do |attributes|
        attributes[:files].each do |fs|
          job = BackupJob.find_or_create_by_server_id_and_hierarchy(server.id, fs)
          job.client_type = "TiNa"
          job.client_version = attributes[:client_version]
          job.catalog = attributes[:catalog]
          job.save if job.changed?
        end
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.1503384396152476, 'fraction_numerical': 0.003562522265764161, 'mean_word_length': 2.338882282996433, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '633508', 'n_tokens_mistral': 892, 'n_tokens_neox': 882, 'n_words': 191}