<commit_before>module Overcommit
  # Provide a set of utilities for certain interactions with `git`.
  module GitRepo
    module_function

    # Regular expression used to extract diff ranges from hunks of diff output.
    DIFF_HUNK_REGEX = /
      ^@@\s
      [^\s]+\s           # Ignore old file range
      \+(\d+)(?:,(\d+))? # Extract range of hunk containing start line and number of lines
      \s@@.*$
    /x

    # Extract the set of modified lines from a given file.
    #
    # @param file_path [String]
    # @param options [Hash]<commit_msg>Add SubmoduleStatus struct, regex and helpers to GitRepo<commit_after>module Overcommit
  # Provide a set of utilities for certain interactions with `git`.
  module GitRepo
    module_function

    # Regular expression used to extract diff ranges from hunks of diff output.
    DIFF_HUNK_REGEX = /
      ^@@\s
      [^\s]+\s           # Ignore old file range
      \+(\d+)(?:,(\d+))? # Extract range of hunk containing start line and number of lines
      \s@@.*$
    /x

    # Regular expression used to extract information from lines of
    # `git submodule status` output
    SUBMODULE_STATUS_REGEX = /
      ^\s*(?<prefix>[-+U]?)(?<sha1>\w+)
      \s(?<path>[^\s]+?)
      (?:\s\((?<describe>.+)\))?$
    /x

    # Struct encapsulating submodule information extracted from the
    # output of `git submodule status`
    SubmoduleStatus = Struct.new(:prefix, :sha1, :path, :describe) do
      # Returns whether the submodule has not been initialized
      def uninitialized?
        prefix == '-'
      end

      # Returns whether the submodule is out of date with the current
      # index, i.e. its checked-out commit differs from that stored in
      # the index of the parent repo
      def outdated?
        prefix == '+'
      end

      # Returns whether the submodule reference has a merge conflict
      def merge_conflict?
        prefix == 'U'
      end
    end

    # Returns a list of SubmoduleStatus objects, one for each submodule in the
    # parent repository.
    #
    # @option options [Boolean] recursive check submodules recursively
    # @return [Array<SubmoduleStatus>]
    def submodule_statuses(options = {})
      flags = '--recursive' if options[:recursive]

      `git submodule status #{flags}`.
        scan(SUBMODULE_STATUS_REGEX).
        map do |prefix, sha1, path, describe|
          SubmoduleStatus.new(prefix, sha1, path, describe)
        end
    end

    # Extract the set of modified lines from a given file.
    #
    # @param file_path [String]
    # @param options [Hash]
{'fraction_non_alphanumeric': 0.10147744945567652, 'fraction_numerical': 0.0015552099533437014, 'mean_word_length': 2.5198358413132693, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7527024', 'n_tokens_mistral': 762, 'n_tokens_neox': 734, 'n_words': 274}