<commit_before>require 'yaml'

module Backup
  class Repository
    attr_reader :repos_path

    def dump
      prepare

      Project.find_each(batch_size: 1000) do |project|
        print " * #{project.path_with_namespace} ... "

        if project.empty_repo?
          puts "[SKIPPED]".cyan
          next
        end

        # Create namespace dir if missing
        FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace

        if system("cd #{path_to_repo(project)} > /dev/null 2>&1 && git bundle create #{path_to_bundle(project)} --all > /dev/null 2>&1")
          puts "[DONE]".green
        else
          puts "[FAILED]".red
        end

        wiki = GollumWiki.new(project)

        if File.exists?(path_to_repo(wiki))
          print " * #{wiki.path_with_namespace} ... "
          if system("cd #{path_to_repo(wiki)} > /dev/null 2>&1 && git bundle create #{path_to_bundle(wiki)} --all > /dev/null 2>&1")
            puts " [DONE]".green
          else
            puts " [FAILED]".red
          end
        end
      end
    end

    def restore
      if File.exists?(repos_path)
        # Move repos dir to 'repositories.old' dir
        bk_repos_path = File.join(repos_path, '..', 'repositories.old.' + Time.now.to_i.to_s)
        FileUtils.mv(repos_path, bk_repos_path)
      end

      FileUtils.mkdir_p(repos_path)

      Project.find_each(batch_size: 1000) do |project|
        print "#{project.path_with_namespace} ... "

        project.namespace.ensure_dir_exist if project.namespace

        if system("git clone --bare #{path_to_bundle(project)} #{path_to_repo(project)} > /dev/null 2>&1")
          puts "[DONE]".green
        else
          puts "[FAILED]".red
        end
<commit_msg>Skip rather than fail empty wikis during backup<commit_after>require 'yaml'

module Backup
  class Repository
    attr_reader :repos_path

    def dump
      prepare

      Project.find_each(batch_size: 1000) do |project|
        print " * #{project.path_with_namespace} ... "

        if project.empty_repo?
          puts "[SKIPPED]".cyan
          next
        end

        # Create namespace dir if missing
        FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace

        if system("cd #{path_to_repo(project)} > /dev/null 2>&1 && git bundle create #{path_to_bundle(project)} --all > /dev/null 2>&1")
          puts "[DONE]".green
        else
          puts "[FAILED]".red
        end

        wiki = GollumWiki.new(project)

        if File.exists?(path_to_repo(wiki))
          print " * #{wiki.path_with_namespace} ... "
          if wiki.empty?
            puts " [SKIPPED]".cyan
          elsif system("cd #{path_to_repo(wiki)} > /dev/null 2>&1 && git bundle create #{path_to_bundle(wiki)} --all > /dev/null 2>&1")
            puts " [DONE]".green
          else
            puts " [FAILED]".red
          end
        end
      end
    end

    def restore
      if File.exists?(repos_path)
        # Move repos dir to 'repositories.old' dir
        bk_repos_path = File.join(repos_path, '..', 'repositories.old.' + Time.now.to_i.to_s)
        FileUtils.mv(repos_path, bk_repos_path)
      end

      FileUtils.mkdir_p(repos_path)

      Project.find_each(batch_size: 1000) do |project|
        print "#{project.path_with_namespace} ... "

        project.namespace.ensure_dir_exist if project.namespace

        if system("git clone --bare #{path_to_bundle(project)} #{path_to_repo(project)} > /dev/null 2>&1")
          puts "[DONE]".green
        else
          puts "[FAILED]".red
        end

{'fraction_non_alphanumeric': 0.12281674521763238, 'fraction_numerical': 0.0099805932908234, 'mean_word_length': 2.162138475021911, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4789481', 'n_tokens_mistral': 1196, 'n_tokens_neox': 1134, 'n_words': 247}