<commit_before>#--
#   Copyright (C) 2006  <NAME>  <andrea (at) rubyforge.org>
#
# This file is part of Maruku.
# 
#   Maruku is free software; you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or
#   (at your option) any later version.
# 
#   Maruku is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
# 
#   You should have received a copy of the GNU General Public License
#   along with Maruku; if not, write to the Free Software
#   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
#++


module MaRuKu; module In; module Markdown; module SpanLevelParser

# a string scanner coded by me
class CharSourceManual; end

# a wrapper around StringScanner
class CharSourceStrscan; end

# A debug scanner that checks the correctness of both
# by comparing their output
class CharSourceDebug; end

# Choose!

#CharSource = CharSourceManual     # faster! 58ms vs. 65ms
CharSource = CharSourceStrscan     # Seems faster on LONG documents (where we care)
#CharSource = CharSourceDebug


class CharSourceManual
	include MaRuKu::Strings
	
	def initialize(s, parent=nil)
		raise "Passed #{s.class}" if not s.kind_of? String
		@buffer = s
		@buffer_index = 0
		@parent = parent
	end
	
	# Return current char as a FixNum (or nil).
	def cur_char; @buffer[@buffer_index]   end

	# Return the next n chars as a String.
	def cur_chars(n); @buffer[@buffer_index,n]  end
	
	# Return the char after current char as a FixNum (or nil).
	def next_char; @buffer[@buffer_index+1] end
	<commit_msg>StringScanner is not multibyte-safe. Avoid like the plague.<commit_after>#--
#   Copyright (C) 2006  <NAME>  <andrea (at) rubyforge.org>
#
# This file is part of Maruku.
# 
#   Maruku is free software; you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or
#   (at your option) any later version.
# 
#   Maruku is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
# 
#   You should have received a copy of the GNU General Public License
#   along with Maruku; if not, write to the Free Software
#   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
#++


module MaRuKu; module In; module Markdown; module SpanLevelParser

# a string scanner coded by me
class CharSourceManual; end

# a wrapper around StringScanner
class CharSourceStrscan; end

# A debug scanner that checks the correctness of both
# by comparing their output
class CharSourceDebug; end

# Choose!

CharSource = CharSourceManual     # faster! 58ms vs. 65ms
#CharSource = CharSourceStrscan   # Faster on LONG documents. But it's not multi-byte aware
                                  # (at least in Ruby 1.9.2 and earlier), Hence unusable in Ruby 1.9.
#CharSource = CharSourceDebug


class CharSourceManual
	include MaRuKu::Strings
	
	def initialize(s, parent=nil)
		raise "Passed #{s.class}" if not s.kind_of? String
		@buffer = s
		@buffer_index = 0
		@parent = parent
	end
	
	# Return current char as a FixNum (or nil).
	def cur_char; @buffer[@buffer_index]   end

	# Return the next n chars as a String.
	def cur_chars(n); @buffer[@buffer_index,n]  end
	
	# Return the char after current char as a FixNum (or nil).
	def next_char; @buffer[@buffer_index+1] end
	
{'fraction_non_alphanumeric': 0.07110639426903688, 'fraction_numerical': 0.013000795967100026, 'mean_word_length': 3.6948941469489416, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7462186', 'n_tokens_mistral': 1198, 'n_tokens_neox': 1104, 'n_words': 506}