<commit_before>##
# The parser module provides standard methods for accessing the headers and
# content of a response that are shared across pluggable parsers.

module Mechanize::Parser

  extend Forwardable

  special_filenames = Regexp.union %w[
    AUX
    COM1
    COM2
    COM3
    COM4
    COM5
    COM6
    COM7
    COM8
    COM9
    CON
    LPT1
    LPT2
    LPT3
    LPT4
    LPT5
    LPT6
    LPT7
    LPT8
    LPT9
    NUL
    PRN
  ]

  ##
  # Special filenames that must be escaped

  SPECIAL_FILENAMES = /\A#{special_filenames}/i

  ##
  # The URI this file was retrieved from

  attr_accessor :uri

  ##
  # The Mechanize::Headers for this file

  attr_accessor :response

  alias header response

  ##
  # The HTTP response code

  attr_accessor :code

  ##
  # :method: [](header)
  #
  # Access HTTP +header+ by name

  def_delegator :header, :[], :[]

  ##
  # :method: []=(header, value)
  #
  # Set HTTP +header+ to +value+

  def_delegator :header, :[]=, :[]=

  ##
  # :method: key?(header)
  #
  # Is the named +header+ present?

  def_delegator :header, :key?, :key?

  ##
  # :method: each
  #
  # Enumerate HTTP headers

  def_delegator :header, :each, :each

  ##
  # :method: each
  #
  # Enumerate HTTP headers in capitalized (canonical) form

  def_delegator :header, :canonical_each, :canonical_each

  ##
  # Extracts the filename from a Content-Disposition header in the #response
  # or from the URI.  If +full_path+ is true the filename will include the
  # host name and path to the resource, otherwise a filename in the current
  # directory is given.

  def extract_filename full_path = @full_path
    handled = false

    if @uri then
      uri = @uri
      uri += 'index.html' if uri.path.end_with? '/'

      path     = uri.path.split(/\//)
      filename = path.pop || 'index.html'
    else
      path     = []
      filename = 'index.html'
    end

    # Set the filename
    if disposition = @response['content-disposition'] then
      content_disposition =
        Mechanize::HTTP::ContentDispositionParser.parse disposition

      if content_disposition then
        filename = content_disposition.filename
        filename = filename.split(/[\\\/]/).last
        handled = true<commit_msg>Don't crash on bad Content-Disposition header.<commit_after>##
# The parser module provides standard methods for accessing the headers and
# content of a response that are shared across pluggable parsers.

module Mechanize::Parser

  extend Forwardable

  special_filenames = Regexp.union %w[
    AUX
    COM1
    COM2
    COM3
    COM4
    COM5
    COM6
    COM7
    COM8
    COM9
    CON
    LPT1
    LPT2
    LPT3
    LPT4
    LPT5
    LPT6
    LPT7
    LPT8
    LPT9
    NUL
    PRN
  ]

  ##
  # Special filenames that must be escaped

  SPECIAL_FILENAMES = /\A#{special_filenames}/i

  ##
  # The URI this file was retrieved from

  attr_accessor :uri

  ##
  # The Mechanize::Headers for this file

  attr_accessor :response

  alias header response

  ##
  # The HTTP response code

  attr_accessor :code

  ##
  # :method: [](header)
  #
  # Access HTTP +header+ by name

  def_delegator :header, :[], :[]

  ##
  # :method: []=(header, value)
  #
  # Set HTTP +header+ to +value+

  def_delegator :header, :[]=, :[]=

  ##
  # :method: key?(header)
  #
  # Is the named +header+ present?

  def_delegator :header, :key?, :key?

  ##
  # :method: each
  #
  # Enumerate HTTP headers

  def_delegator :header, :each, :each

  ##
  # :method: each
  #
  # Enumerate HTTP headers in capitalized (canonical) form

  def_delegator :header, :canonical_each, :canonical_each

  ##
  # Extracts the filename from a Content-Disposition header in the #response
  # or from the URI.  If +full_path+ is true the filename will include the
  # host name and path to the resource, otherwise a filename in the current
  # directory is given.

  def extract_filename full_path = @full_path
    handled = false

    if @uri then
      uri = @uri
      uri += 'index.html' if uri.path.end_with? '/'

      path     = uri.path.split(/\//)
      filename = path.pop || 'index.html'
    else
      path     = []
      filename = 'index.html'
    end

    # Set the filename
    if disposition = @response['content-disposition'] then
      content_disposition =
        Mechanize::HTTP::ContentDispositionParser.parse disposition

      if content_disposition && content_disposition.filename then
        filename = content_disposition.filename
        filename = filename.split(/[\\\/]/).last
        handled = true
{'fraction_non_alphanumeric': 0.09737827715355805, 'fraction_numerical': 0.007931262392597489, 'mean_word_length': 2.56078431372549, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5488968', 'n_tokens_mistral': 1524, 'n_tokens_neox': 1464, 'n_words': 443}