<commit_before>class Repositext

  # Represents a collection of git content and code repositories.
  # Assumes that all repositories are siblings in the same folder.
  #
  # Expects current directory to be a repositext content repo root path.
  #
  # Usage example:
  #     repository_set = RepositorySet.new('/repositories/parent/path')
  #     repository_set.git_pull(:all_content_repos)
  class RepositorySet

    attr_reader :repo_set_parent_path

    # @param repo_set_parent_path [String] path to the folder that contains all repos.
    def initialize(repo_set_parent_path)
      @repo_set_parent_path = repo_set_parent_path
    end

    def all_content_repo_names
      [primary_repo_name] + foreign_content_repo_names
    end

    def all_repo_names
      all_content_repo_names + code_repo_names
    end

    # Returns an array of paths to all repos in repo_set_spec
    # @param repo_set_spec [Symbol, Array<String>] A symbol describing a predefined
    #     group of repos, or an Array with specific repo names as strings.
    def all_repo_paths(repo_set_spec)
      compute_repo_paths(repo_set_spec)
    end

    # Returns an array of all repos in repo_set
    def all_repos(repo_set)
      repo_class = if repo_set.is_a?(Symbol) && repo_set.to_s.include?('_content_')
        Repository::Content
      else
        Repository
      end
      all_repo_paths(repo_set).map{ |e| repo_class.new(e) }
    end

    def code_repo_names
      %w[
        repositext
        suspension
      ]
    end

    # Run `delete_all_pdf_exports` in all repositories
    def delete_all_pdf_exports(repo_set_spec, content_type)
      run_repositext_command(repo_set_spec, "rt #{ content_type.name } delete all_pdf_exports")
    end

    # Run `export pdf_book` in all repositories
    def export_pdf_book(repo_set_spec, content_type)
      run_repositext_command(repo_set_spec, "rt #{ content_type.name } export pdf_book")
    end

    def fix_add_initial_data_json_file(repo_set_spec, content_type)
      run_repositext_command(repo_set_spec, "rt #{ content_type.name } fix add_initial_data_json_file")
    end

    def fix_normalize_trailing_newlines(repo_set_spec, content_type)
      run_repositext_command(repo_set_spec, "rt #{ content_type.name } fix normalize_trailing_newlines")
    end

    def foreign_content_repo_names
      %w[
        french
        german
        italian
        spanish
      ]
    end

    # Extracting the git clone command so that it can be overridden in subclasses.
    # @param repo_name [String]
    def git_clone_command(repo_name)
      "git clone <repo clone url>#{ repo_name }.git"
    end

    # Clones all git repos that don't exist on local filesystem yet.
    # @param repo_set_spec [Symbol, Array<String>] A symbol describing a predefined
    #     group of repos, or an Array with specific repo names as strings.
    # @example Clone all language repos
    #   # cd into primary repo root folder
    #   # run `bundle console`
    #   repository_set = Repositext::RepositorySet.new('/path/to/repos/parent/folder')
    #   repository_set.git_clone_missing_repos(:all_content_repos)
    def git_clone_missing_repos(repo_set_spec)
      compute_repo_paths(repo_set_spec).each { |repo_path|
        repo_name = repo_path.split('/').last
        if File.exists?(repo_path)
          puts " -   Skipping #{ repo_name }"
          next
        end
        puts " - Cloning #{ repo_name }"
        clone_command = git_clone_command(repo_name)
        cmd = %(cd #{ repo_set_parent_path } && #{ clone_command })
        Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr|
          exit_status = wait_thr.value
          if exit_status.success?
            puts "   - Cloned #{ repo_name }"
          else
            msg = %(Could not clone #{ repo_name }:\n\n)
            puts(msg + stderr.read)
          end
        end
      }
    end

    # Makes sure that all content repos are ready for git operations:
    # * They are on master branch
    # * They have no uncommitted changes
    # * They pulled latest from origin
    # @param repo_set_spec [Symbol, Array<String>] A symbol describing a predefined
    #     group of repos, or an Array with specific repo names as strings.
    # @yield [repo_path] will be called for each repo.
    # @return [Hash] with repos that are not ready. Keys are repo paths, values
    #     are arrays with issue messages if any exist.
    def git_ensure_repos_are_ready(repo_set_spec)
      # TODO: when we use this from st_sync we only want to include repos that have
      # st_sync_active set to true. We already have a method that gives us a list of
      # all synced repos. Maybe we could move this check to Repository... Then we
      # could use it from here, or the other collection.
      repos_with_issues = {}
      compute_repo_paths(repo_set_spec).each { |repo_path|
        if block_given?
          yield(repo_path)
        end
        repo_issues = []
        cmd = %(cd #{ repo_path } && git pull && git status)
        Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr|
          exit_status = wait_thr.value
          r = stdout.read
          if !r.index('On branch master')
            repo_issues << "Is not on master branch."
          end
          if !r.index(%(Your branch is up-to-date with 'origin/master'))
            repo_issues << "Is not up-to-date with origin"
          end
          if !(
            r.index(%(nothing to commit, working directory clean)) ||
            r.index(%(nothing added to commit but untracked files present))
          )
            repo_issues << "Has uncommitted changes"
          end
          if !exit_status.success?
            repo_issues << "Error: could not check repo (#{ stderr.read })".color(:red)
          end
        end
        if repo_issues.any?
          repos_with_issues[repo_path] = repo_issues
        end
      }
      repos_with_issues
    end

    # Fetches all branches in all repos and pulls current branch
    # @param repo_set_spec [Symbol, Array<String>] A symbol describing a predefined
    #     group of repos, or an Array with specific repo names as strings.
    def git_fetch_and_pull(repo_set_spec)
      compute_repo_paths(repo_set_spec).each { |repo_path|
        cmd = %(cd #{ repo_path } && git fetch && git pull)
        Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr|
          exit_status = wait_thr.value
          if exit_status.success?
            puts " - Fetched and pulled #{ repo_path }"
          else
            msg = %(Could not fetch and pull ull #{ repo_path }:\n\n).color(:red)
            puts(msg + stderr.read)
          end
        end
      }
    end

    # Pulls all repos
    # @param repo_set_spec [Symbol, Array<String>] A symbol describing a predefined
    #     group of repos, or an Array with specific repo names as strings.
    def git_pull(repo_set_spec)
      compute_repo_paths(repo_set_spec).each { |repo_path|
        cmd = %(cd #{ repo_path } && git pull)
        Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr|
          exit_status = wait_thr.value
          if exit_status.success?
            puts " - Pulled #{ repo_path }"
          else
            msg = %(Could not pull #{ repo_path }:\n\n).color(:red)
            puts(msg + stderr.read)
          end<commit_msg>Updated git_repo command git_fetch_and_pull:<commit_after>class Repositext

  # Represents a collection of git content and code repositories.
  # Assumes that all repositories are siblings in the same folder.
  #
  # Expects current directory to be a repositext content repo root path.
  #
  # Usage example:
  #     repository_set = RepositorySet.new('/repositories/parent/path')
  #     repository_set.git_pull(:all_content_repos)
  class RepositorySet

    attr_reader :repo_set_parent_path

    # @param repo_set_parent_path [String] path to the folder that contains all repos.
    def initialize(repo_set_parent_path)
      @repo_set_parent_path = repo_set_parent_path
    end

    def all_content_repo_names
      [primary_repo_name] + foreign_content_repo_names
    end

    def all_repo_names
      all_content_repo_names + code_repo_names
    end

    # Returns an array of paths to all repos in repo_set_spec
    # @param repo_set_spec [Symbol, Array<String>] A symbol describing a predefined
    #     group of repos, or an Array with specific repo names as strings.
    def all_repo_paths(repo_set_spec)
      compute_repo_paths(repo_set_spec)
    end

    # Returns an array of all repos in repo_set
    def all_repos(repo_set)
      repo_class = if repo_set.is_a?(Symbol) && repo_set.to_s.include?('_content_')
        Repository::Content
      else
        Repository
      end
      all_repo_paths(repo_set).map{ |e| repo_class.new(e) }
    end

    def code_repo_names
      %w[
        repositext
        suspension
      ]
    end

    # Run `delete_all_pdf_exports` in all repositories
    def delete_all_pdf_exports(repo_set_spec, content_type)
      run_repositext_command(repo_set_spec, "rt #{ content_type.name } delete all_pdf_exports")
    end

    # Run `export pdf_book` in all repositories
    def export_pdf_book(repo_set_spec, content_type)
      run_repositext_command(repo_set_spec, "rt #{ content_type.name } export pdf_book")
    end

    def fix_add_initial_data_json_file(repo_set_spec, content_type)
      run_repositext_command(repo_set_spec, "rt #{ content_type.name } fix add_initial_data_json_file")
    end

    def fix_normalize_trailing_newlines(repo_set_spec, content_type)
      run_repositext_command(repo_set_spec, "rt #{ content_type.name } fix normalize_trailing_newlines")
    end

    def foreign_content_repo_names
      %w[
        french
        german
        italian
        spanish
      ]
    end

    # Extracting the git clone command so that it can be overridden in subclasses.
    # @param repo_name [String]
    def git_clone_command(repo_name)
      "git clone <repo clone url>#{ repo_name }.git"
    end

    # Clones all git repos that don't exist on local filesystem yet.
    # @param repo_set_spec [Symbol, Array<String>] A symbol describing a predefined
    #     group of repos, or an Array with specific repo names as strings.
    # @example Clone all language repos
    #   # cd into primary repo root folder
    #   # run `bundle console`
    #   repository_set = Repositext::RepositorySet.new('/path/to/repos/parent/folder')
    #   repository_set.git_clone_missing_repos(:all_content_repos)
    def git_clone_missing_repos(repo_set_spec)
      compute_repo_paths(repo_set_spec).each { |repo_path|
        repo_name = repo_path.split('/').last
        if File.exists?(repo_path)
          puts " -   Skipping #{ repo_name }"
          next
        end
        puts " - Cloning #{ repo_name }"
        clone_command = git_clone_command(repo_name)
        cmd = %(cd #{ repo_set_parent_path } && #{ clone_command })
        Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr|
          exit_status = wait_thr.value
          if exit_status.success?
            puts "   - Cloned #{ repo_name }"
          else
            msg = %(Could not clone #{ repo_name }:\n\n)
            puts(msg + stderr.read)
          end
        end
      }
    end

    # Makes sure that all content repos are ready for git operations:
    # * They are on master branch
    # * They have no uncommitted changes
    # * They pulled latest from origin
    # @param repo_set_spec [Symbol, Array<String>] A symbol describing a predefined
    #     group of repos, or an Array with specific repo names as strings.
    # @yield [repo_path] will be called for each repo.
    # @return [Hash] with repos that are not ready. Keys are repo paths, values
    #     are arrays with issue messages if any exist.
    def git_ensure_repos_are_ready(repo_set_spec)
      # TODO: when we use this from st_sync we only want to include repos that have
      # st_sync_active set to true. We already have a method that gives us a list of
      # all synced repos. Maybe we could move this check to Repository... Then we
      # could use it from here, or the other collection.
      repos_with_issues = {}
      compute_repo_paths(repo_set_spec).each { |repo_path|
        if block_given?
          yield(repo_path)
        end
        repo_issues = []
        cmd = %(cd #{ repo_path } && git pull && git status)
        Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr|
          exit_status = wait_thr.value
          r = stdout.read
          if !r.index('On branch master')
            repo_issues << "Is not on master branch."
          end
          if !r.index(%(Your branch is up-to-date with 'origin/master'))
            repo_issues << "Is not up-to-date with origin"
          end
          if !(
            r.index(%(nothing to commit, working directory clean)) ||
            r.index(%(nothing added to commit but untracked files present))
          )
            repo_issues << "Has uncommitted changes"
          end
          if !exit_status.success?
            repo_issues << "Error: could not check repo (#{ stderr.read })".color(:red)
          end
        end
        if repo_issues.any?
          repos_with_issues[repo_path] = repo_issues
        end
      }
      repos_with_issues
    end

    # Fetches all branches (including new ones on remote) in all repos and pulls
    # all local tracking branches.
    # @param repo_set_spec [Symbol, Array<String>] A symbol describing a predefined
    #     group of repos, or an Array with specific repo names as strings.
    def git_fetch_and_pull(repo_set_spec)
      compute_repo_paths(repo_set_spec).each { |repo_path|
        cmd = %(cd #{ repo_path } && git remote update && git pull --all)
        Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr|
          exit_status = wait_thr.value
          if exit_status.success?
            puts " - Fetched and pulled #{ repo_path }"
          else
            msg = %(Could not fetch and pull #{ repo_path }:\n\n).color(:red)
            puts(msg + stderr.read)
          end
        end
      }
    end

    # Pulls all repos
    # @param repo_set_spec [Symbol, Array<String>] A symbol describing a predefined
    #     group of repos, or an Array with specific repo names as strings.
    def git_pull(repo_set_spec)
      compute_repo_paths(repo_set_spec).each { |repo_path|
        cmd = %(cd #{ repo_path } && git pull)
        Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr|
          exit_status = wait_thr.value
          if exit_status.success?
            puts " - Pulled #{ repo_path }"
          else
            msg = %(Could not pull #{ repo_path }:\n\n).color(:red)
            puts(msg + stderr.read)
          end
{'fraction_non_alphanumeric': 0.07807828214188638, 'fraction_numerical': 0.0010872519706441968, 'mean_word_length': 2.5107347328244276, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2209038', 'n_tokens_mistral': 4567, 'n_tokens_neox': 4436, 'n_words': 1318}