<commit_before># Copyright (c) 2013, HipByte SPRL and Contributors
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# Special thanks to https://github.com/tenderlove/psych
class YAMLKitScanner
  # http://yaml.org/type/timestamp.html
  TIME = /^-?\d{4}-\d{1,2}-\d{1,2}(?:[Tt]|\s+)\d{1,2}:\d\d:\d\d(?:\.\d*)?(?:\s*(?:Z|[-+]\d{1,2}:?(?:\d\d)?))?$/

  # http://yaml.org/type/float.html
  FLOAT = /^(?:[-+]?([0-9][0-9_,]*)?\.[0-9]*([eE][-+][0-9]+)?(?# base 10)
            |[-+]?[0-9][0-9_,]*(:[0-5]?[0-9])+\.[0-9_]*(?# base 60)
            |[-+]?\.(inf|Inf|INF)(?# infinity)
            |\.(nan|NaN|NAN)(?# not a number))$/x

  # http://yaml.org/type/int.html
  INTEGER = /^(?:[-+]?0b[0-1_]+          (?# base 2)
                |[-+]?0[0-7_]+           (?# base 8)
                |[-+]?(?:0|[1-9][0-9_]*) (?# base 10)
                |[-+]?0x[0-9a-fA-F_]+    (?# base 16))$/x

  # Tokenize string returning the Ruby object
  # NOTE: This method will be called from Objective-C.
  def self.tokenize(string)
    case string
    when /^\+?\.inf$/i
      Float::INFINITY
    when /^-\.inf$/i
      -Float::INFINITY
    when /^\.nan$/i
      Float::NAN
    when /^:(.*)/
      string = $1
      if string =~ /^["'](.+)["']$/
        string = $1
      end
      string.to_sym
    when INTEGER
      Integer(string.gsub(/_/, ''))
    when FLOAT
      Float(string.gsub(/_/, ''))
    when TIME
      self.parse_time(string)
    when /^[-+]?[0-9][0-9_]*(:[0-5]?[0-9])+$/
      i = 0
      string.split(':').each_with_index do |n,e|
        i += (n.to_i * 60 ** (e - 2).abs)
      end
      i
    when /^(yes|y|true|on)$/i
      true
    when /^(no|n|false|off)$/i
      false
    when '~'
      nil
    else
      string
    end
  end

  def self.parse_time string<commit_msg>add supporting of floating point in sexagesimal<commit_after># Copyright (c) 2013, HipByte SPRL and Contributors
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# Special thanks to https://github.com/tenderlove/psych
class YAMLKitScanner
  # http://yaml.org/type/timestamp.html
  TIME = /^-?\d{4}-\d{1,2}-\d{1,2}(?:[Tt]|\s+)\d{1,2}:\d\d:\d\d(?:\.\d*)?(?:\s*(?:Z|[-+]\d{1,2}:?(?:\d\d)?))?$/

  # http://yaml.org/type/float.html
  FLOAT = /^(?:[-+]?([0-9][0-9_,]*)?\.[0-9]*([eE][-+][0-9]+)?(?# base 10)
            |[-+]?[0-9][0-9_,]*(:[0-5]?[0-9])+\.[0-9_]*(?# base 60)
            |[-+]?\.(inf|Inf|INF)(?# infinity)
            |\.(nan|NaN|NAN)(?# not a number))$/x

  # http://yaml.org/type/int.html
  INTEGER = /^(?:[-+]?0b[0-1_]+          (?# base 2)
                |[-+]?0[0-7_]+           (?# base 8)
                |[-+]?(?:0|[1-9][0-9_]*) (?# base 10)
                |[-+]?0x[0-9a-fA-F_]+    (?# base 16))$/x

  # Tokenize string returning the Ruby object
  # NOTE: This method will be called from Objective-C.
  def self.tokenize(string)
    case string
    when /^\+?\.inf$/i
      Float::INFINITY
    when /^-\.inf$/i
      -Float::INFINITY
    when /^\.nan$/i
      Float::NAN
    when /^:(.*)/
      string = $1
      if string =~ /^["'](.+)["']$/
        string = $1
      end
      string.to_sym
    when /^[-+]?[0-9][0-9_]*(:[0-5]?[0-9])+$/
      i = 0
      string.split(':').each_with_index do |n,e|
        i += (n.to_i * 60 ** (e - 2).abs)
      end
      i
    when /^[-+]?[0-9][0-9_]*(:[0-5]?[0-9])+\.[0-9_]*$/
      i = 0
      string.split(':').each_with_index do |n,e|
        i += (n.to_f * 60 ** (e - 2).abs)
      end
      i
    when INTEGER
      Integer(string.gsub(/_/, ''))
    when FLOAT
      Float(string.gsub(/_/, ''))
    when TIME
      self.parse_time(string)
    when /^(yes|y|true|on)$/i
      true
    when /^(no|n|false|off)$/i
      false
    when '~'
      nil
    else
      string
    end
  end

  def self.parse_time string
{'fraction_non_alphanumeric': 0.17551150895140666, 'fraction_numerical': 0.024936061381074168, 'mean_word_length': 2.6697947214076247, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6777503', 'n_tokens_mistral': 2349, 'n_tokens_neox': 2131, 'n_words': 633}