<commit_before>require 'stringio'
require 'ostruct'

begin require 'continuation'; rescue LoadError; end

module YARD
  module Parser
    # Raised when an object is recognized but cannot be documented. This
    # generally occurs when the Ruby syntax used to declare an object is
    # too dynamic in nature.
    class UndocumentableError < Exception; end

    # Raised when the parser sees a Ruby syntax error
    class ParserSyntaxError < UndocumentableError; end

    # A LoadOrderError occurs when a handler needs to modify a
    # {CodeObjects::NamespaceObject} (usually by adding a child to it)
    # that has not yet been resolved.
    #
    # @see Handers::Base#ensure_loaded!
    class LoadOrderError < Exception; end

    # Responsible for parsing a source file into the namespace. Parsing
    # also invokes handlers to process the parsed statements and generate
    # any code objects that may be recognized.
    #
    # == Custom Parsers
    # SourceParser allows custom parsers to be registered and called when
    # a certain filetype is recognized. To register a parser and hook it
    # up to a set of file extensions, call {register_parser_type}
    #
    # @see register_parser_type
    # @see Handlers::Base
    # @see CodeObjects::Base
    class SourceParser
      SHEBANG_LINE  = /\A\s*#!\S+/
      ENCODING_LINE = /\A(?:\s*#*!.*\r?\n)?\s*#+.*coding\s*[:=]{1,2}\s*(\S+)/i
      
      # Byte order marks for various encodings
      # @since 0.7.0
      ENCODING_BYTE_ORDER_MARKS = {
        'utf-8' => "\xEF\xBB\xBF",
        # Not yet supported
        #'utf-16be' => "\xFE\xFF",
        #'utf-16le' => "\xFF\xFE",
        #'utf-32be' => "\x00\x00\xFF\xFE",
        #'utf-32le' => "\xFF\xFE",
      }

      class << self
        # @return [Symbol] the default parser type (defaults to :ruby)
        attr_reader :parser_type

        def parser_type=(value)
          @parser_type = validated_parser_type(value)
        end

        # Parses a path or set of paths
        #
        # @param [String, Array<String>] paths a path, glob, or list of paths to
        #   parse
        # @param [Array<String, Regexp>] excluded a list of excluded path matchers
        # @param [Fixnum] level the logger level to use during parsing. See
        #   {YARD::Logger}
        # @return the parser object that was used to parse the source.
        def parse(paths = ["{lib,app}/**/*.rb", "ext/**/*.c"], excluded = [], level = log.level)
          log.debug("Parsing #{paths.inspect} with `#{parser_type}` parser")
          excluded = excluded.map do |path|
            case path
            when Regexp; path
            else Regexp.new(path.to_s, Regexp::IGNORECASE)
            end
          end
          files = [paths].flatten.
            map {|p| File.directory?(p) ? "#{p}/**/*.{rb,c}" : p }.
            map {|p| p.include?("*") ? Dir[p] : p }.flatten.
            reject {|p| !File.file?(p) || excluded.any? {|re| p =~ re } }

          log.enter_level(level) do
            parse_in_order(*files.uniq)
          end
        end

        # Parses a string +content+
        #
        # @param [String] content the block of code to parse
        # @param [Symbol] ptype the parser type to use. See {parser_type}.
        # @return the parser object that was used to parse +content+
        def parse_string(content, ptype = parser_type)
          new(ptype).parse(StringIO.new(content))
        end

        # Tokenizes but does not parse the block of code
        #
        # @param [String] content the block of code to tokenize
        # @param [Symbol] ptype the parser type to use. See {parser_type}.<commit_msg>Fix documentation claiming that SourceParser.parse returns a parser object; it returns void.<commit_after>require 'stringio'
require 'ostruct'

begin require 'continuation'; rescue LoadError; end

module YARD
  module Parser
    # Raised when an object is recognized but cannot be documented. This
    # generally occurs when the Ruby syntax used to declare an object is
    # too dynamic in nature.
    class UndocumentableError < Exception; end

    # Raised when the parser sees a Ruby syntax error
    class ParserSyntaxError < UndocumentableError; end

    # A LoadOrderError occurs when a handler needs to modify a
    # {CodeObjects::NamespaceObject} (usually by adding a child to it)
    # that has not yet been resolved.
    #
    # @see Handers::Base#ensure_loaded!
    class LoadOrderError < Exception; end

    # Responsible for parsing a source file into the namespace. Parsing
    # also invokes handlers to process the parsed statements and generate
    # any code objects that may be recognized.
    #
    # == Custom Parsers
    # SourceParser allows custom parsers to be registered and called when
    # a certain filetype is recognized. To register a parser and hook it
    # up to a set of file extensions, call {register_parser_type}
    #
    # @see register_parser_type
    # @see Handlers::Base
    # @see CodeObjects::Base
    class SourceParser
      SHEBANG_LINE  = /\A\s*#!\S+/
      ENCODING_LINE = /\A(?:\s*#*!.*\r?\n)?\s*#+.*coding\s*[:=]{1,2}\s*(\S+)/i
      
      # Byte order marks for various encodings
      # @since 0.7.0
      ENCODING_BYTE_ORDER_MARKS = {
        'utf-8' => "\xEF\xBB\xBF",
        # Not yet supported
        #'utf-16be' => "\xFE\xFF",
        #'utf-16le' => "\xFF\xFE",
        #'utf-32be' => "\x00\x00\xFF\xFE",
        #'utf-32le' => "\xFF\xFE",
      }

      class << self
        # @return [Symbol] the default parser type (defaults to :ruby)
        attr_reader :parser_type

        def parser_type=(value)
          @parser_type = validated_parser_type(value)
        end

        # Parses a path or set of paths
        #
        # @param [String, Array<String>] paths a path, glob, or list of paths to
        #   parse
        # @param [Array<String, Regexp>] excluded a list of excluded path matchers
        # @param [Fixnum] level the logger level to use during parsing. See
        #   {YARD::Logger}
        # @return [void]
        def parse(paths = ["{lib,app}/**/*.rb", "ext/**/*.c"], excluded = [], level = log.level)
          log.debug("Parsing #{paths.inspect} with `#{parser_type}` parser")
          excluded = excluded.map do |path|
            case path
            when Regexp; path
            else Regexp.new(path.to_s, Regexp::IGNORECASE)
            end
          end
          files = [paths].flatten.
            map {|p| File.directory?(p) ? "#{p}/**/*.{rb,c}" : p }.
            map {|p| p.include?("*") ? Dir[p] : p }.flatten.
            reject {|p| !File.file?(p) || excluded.any? {|re| p =~ re } }

          log.enter_level(level) do
            parse_in_order(*files.uniq)
          end
        end

        # Parses a string +content+
        #
        # @param [String] content the block of code to parse
        # @param [Symbol] ptype the parser type to use. See {parser_type}.
        # @return the parser object that was used to parse +content+
        def parse_string(content, ptype = parser_type)
          new(ptype).parse(StringIO.new(content))
        end

        # Tokenizes but does not parse the block of code
        #
        # @param [String] content the block of code to tokenize
        # @param [Symbol] ptype the parser type to use. See {parser_type}.
{'fraction_non_alphanumeric': 0.11234112341123412, 'fraction_numerical': 0.004920049200492005, 'mean_word_length': 2.284560143626571, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6416555', 'n_tokens_mistral': 2242, 'n_tokens_neox': 2179, 'n_words': 745}