<commit_before>module Ekylibre
  module FirstRun
    class Base
      LOADERS = YAML.load_file(Pathname.new(__FILE__).dirname.join('loaders.yml')).deep_symbolize_keys.freeze

      attr_reader :max, :path, :verbose, :mode, :force

      def initialize(path, options = {})
        @verbose = !options[:verbose].is_a?(FalseClass)
        @force = options[:force].is_a?(TrueClass)
        @mode = options[:mode].to_s.downcase
        @mode = 'normal' if @mode.blank?
        @mode = @mode.to_sym
        @path = path
        unless @path.exist?
          raise ArgumentError, "Need a valid folder path. #{@path} doesn't exist."
        end
        @term_width = begin
                        `/usr/bin/env tput cols`.to_i
                      rescue
                        80
                      end
        @term_width = 80 unless @term_width > 0
        ::I18n.locale = Preference[:language]
        @max = options[:max].to_i
      end

      # Compute a path for first run directory
      def path(*args)
        @path.join(*args)
      end

      def hard?
        @mode == :hard || Rails.env.production?
      end

      def executed_preference
        Preference.get('first_run.executed', false)
      end

      # Execute all loaders for a given base
      def run
        secure_transaction(!hard?) do
          LOADERS.each do |loader, imports|
            run_loader(loader, imports)
          end
          executed_preference.update!(value: true)
        end
      end

      # Execute given loader for a given base
      def run_loader(loader, imports)
        ::I18n.locale = Preference[:language]
        ActiveRecord::Base.transaction do
          preference = Preference.get("first_run.executed_loaders.#{loader}", false)
          if force || !preference.value
            # @loaders[loader].call(base)
            imports.each do |import_name, options|
              format = options[:format] || :file
              path = options[:path]
              nature = options[:type] || import_name
              if format == :archive
                import_archive(nature, path, options[:files], options.slice(:in, :mimetype))
              elsif format == :pictures
                import_pictures(path, options[:table], options[:id_column])
              elsif format == :file
                import_file(nature, path, options)
              else
                raise 'Cannot import that format: ' + format.inspect
              end
            end
            preference.value = true
            preference.save!
          else
            puts 'Skip'.yellow + " #{loader} loader"
          end
        end
      end

      # Wrap code in a transaction if wanted
      def secure_transaction(with_transaction = true, &block)
        (with_transaction ? ActiveRecord::Base.transaction(&block) : yield)
      end

      def import_pictures(base, type, identifier)
        if path("#{base}/pictures").exist?
          file = path("#{base}/pictures.zip")
          FileUtils.rm_rf file

          mimefile = path.join("#{base}/pictures.mimetype")
          File.write(mimefile, "application/vnd.ekylibre.pictures.#{type}")

          idenfile = path.join("#{base}/pictures.identifier")
          File.write(idenfile, identifier.to_s)

          files = {
            'mimetype' => 'pictures.mimetype',
            'identifier' => 'pictures.identifier'
          }
          Dir.chdir(path(base.to_s)) do
            Dir.glob('pictures/*').each do |picture|
              p = Pathname.new(picture).basename.to_s
              files[picture] = "pictures/#{p}"
            end
          end
          check_archive(:ekylibre_pictures, file, files, in: base.to_s)

          FileUtils.rm_f mimefile
          FileUtils.rm_f idenfile
        end
        import_file(:ekylibre_pictures, "#{base}/pictures.zip")
      end

      def import_file(nature, file, options = {})
        p = path(file)
        options = options.map do |key, value|
          next [key, value] unless key.to_s =~ /_path$/
          [key, path(value).to_s]
        end.to_h
        if p.exist?
          import(nature, p, options)
        elsif @verbose
          text = [" ∅ #{nature.to_s.humanize}", " (#{p.basename})"]
          if text.join.length < @term_width
            text << ' ' * (@term_width - text.join.length)
          end
          text[0] = text[0].yellow
          puts text.join
        end
      end

      def import_archive(nature, target, files, options = {})
        file = check_archive(nature, target, files, options)
        import_file(nature, file) if file
      end

      # Check that archive exist if not try to build one if existing file
      # Given files must exist
      def check_archive(nature, target, files, options = {})
        working_path = @path.join(options[:in] ? options[:in] : '.')
        target_path = working_path.join(target)
        files = files.each_with_object({}) { |f, h| h[f] = f } if files.is_a?(Array)
        files.each do |dest, source|
          files[dest] = working_path.join(source)
        end
        files_count = files.keys.count
        mimefile = working_path.join('mimetype')
        FileUtils.rm_rf(target_path) if target_path.exist?
        not_found = files.values.reject(&:exist?)
        if not_found.any?
          if not_found.size != files_count
            puts " ☠ #{nature.to_s.humanize} (#{target})".red
            puts ("   Missing files to build #{target} archive: " + not_found.map { |f| f.relative_path_from(@path) }.to_sentence).red<commit_msg>Setting back delete where it was used.<commit_after>module Ekylibre
  module FirstRun
    class Base
      LOADERS = YAML.load_file(Pathname.new(__FILE__).dirname.join('loaders.yml')).deep_symbolize_keys.freeze

      attr_reader :max, :path, :verbose, :mode, :force

      def initialize(path, options = {})
        @verbose = !options[:verbose].is_a?(FalseClass)
        @force = options[:force].is_a?(TrueClass)
        @mode = options[:mode].to_s.downcase
        @mode = 'normal' if @mode.blank?
        @mode = @mode.to_sym
        @path = path
        unless @path.exist?
          raise ArgumentError, "Need a valid folder path. #{@path} doesn't exist."
        end
        @term_width = begin
                        `/usr/bin/env tput cols`.to_i
                      rescue
                        80
                      end
        @term_width = 80 unless @term_width > 0
        ::I18n.locale = Preference[:language]
        @max = options[:max].to_i
      end

      # Compute a path for first run directory
      def path(*args)
        @path.join(*args)
      end

      def hard?
        @mode == :hard || Rails.env.production?
      end

      def executed_preference
        Preference.get('first_run.executed', false)
      end

      # Execute all loaders for a given base
      def run
        secure_transaction(!hard?) do
          LOADERS.each do |loader, imports|
            run_loader(loader, imports)
          end
          executed_preference.update!(value: true)
        end
      end

      # Execute given loader for a given base
      def run_loader(loader, imports)
        ::I18n.locale = Preference[:language]
        ActiveRecord::Base.transaction do
          preference = Preference.get("first_run.executed_loaders.#{loader}", false)
          if force || !preference.value
            # @loaders[loader].call(base)
            imports.each do |import_name, options|
              format = options[:format] || :file
              path = options[:path]
              nature = options[:type] || import_name
              if format == :archive
                import_archive(nature, path, options[:files], options.slice(:in, :mimetype))
              elsif format == :pictures
                import_pictures(path, options[:table], options[:id_column])
              elsif format == :file
                import_file(nature, path, options)
              else
                raise 'Cannot import that format: ' + format.inspect
              end
            end
            preference.value = true
            preference.save!
          else
            puts 'Skip'.yellow + " #{loader} loader"
          end
        end
      end

      # Wrap code in a transaction if wanted
      def secure_transaction(with_transaction = true, &block)
        (with_transaction ? ActiveRecord::Base.transaction(&block) : yield)
      end

      def import_pictures(base, type, identifier)
        if path("#{base}/pictures").exist?
          file = path("#{base}/pictures.zip")
          FileUtils.rm_rf file

          mimefile = path.join("#{base}/pictures.mimetype")
          File.write(mimefile, "application/vnd.ekylibre.pictures.#{type}")

          idenfile = path.join("#{base}/pictures.identifier")
          File.write(idenfile, identifier.to_s)

          files = {
            'mimetype' => 'pictures.mimetype',
            'identifier' => 'pictures.identifier'
          }
          Dir.chdir(path(base.to_s)) do
            Dir.glob('pictures/*').each do |picture|
              p = Pathname.new(picture).basename.to_s
              files[picture] = "pictures/#{p}"
            end
          end
          check_archive(:ekylibre_pictures, file, files, in: base.to_s)

          FileUtils.rm_f mimefile
          FileUtils.rm_f idenfile
        end
        import_file(:ekylibre_pictures, "#{base}/pictures.zip")
      end

      def import_file(nature, file, options = {})
        p = path(file)
        options = options.map do |key, value|
          next [key, value] unless key.to_s =~ /_path$/
          [key, path(value).to_s]
        end.to_h
        if p.exist?
          import(nature, p, options)
        elsif @verbose
          text = [" ∅ #{nature.to_s.humanize}", " (#{p.basename})"]
          if text.join.length < @term_width
            text << ' ' * (@term_width - text.join.length)
          end
          text[0] = text[0].yellow
          puts text.join
        end
      end

      def import_archive(nature, target, files, options = {})
        file = check_archive(nature, target, files, options)
        import_file(nature, file) if file
      end

      # Check that archive exist if not try to build one if existing file
      # Given files must exist
      def check_archive(nature, target, files, options = {})
        working_path = @path.join(options[:in] ? options.delete(:in) : '.')
        target_path = working_path.join(target)
        files = files.each_with_object({}) { |f, h| h[f] = f } if files.is_a?(Array)
        files.each do |dest, source|
          files[dest] = working_path.join(source)
        end
        files_count = files.keys.count
        mimefile = working_path.join('mimetype')
        FileUtils.rm_rf(target_path) if target_path.exist?
        not_found = files.values.reject(&:exist?)
        if not_found.any?
          if not_found.size != files_count
            puts " ☠ #{nature.to_s.humanize} (#{target})".red
            puts ("   Missing files to build #{target} archive: " + not_found.map { |f| f.relative_path_from(@path) }.to_sentence).red
{'fraction_non_alphanumeric': 0.11470375722543352, 'fraction_numerical': 0.001986994219653179, 'mean_word_length': 1.9285903200211585, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2337774', 'n_tokens_mistral': 3305, 'n_tokens_neox': 3200, 'n_words': 749}