<commit_before>#!/usr/bin/env ruby
#
# State Based Session Management	
#	Copyright (C) 2004 <NAME>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#	ywesee - intellectual capital connected, Winterthurerstrasse 52, CH-8006 Zrich, Switzerland
#	<EMAIL>
#
# TestValidator -- sbsm -- 15.11.2002 -- <EMAIL> 

$: << File.dirname(__FILE__)
$: << File.expand_path("../lib", File.dirname(__FILE__))

require 'test/unit'
require 'sbsm/validator'

class Validator < SBSM::Validator
	attr_accessor :errors
	DATES = [
		:date
	]
	ENUMS = {
		:enum	=>	['foo'],
		:values =>	['1', '2', '3', '4', '7'],
	}
	EVENTS = [
		:logout,
	]
	BOOLEAN = [ :bool ]
	PATTERNS = {
		:pattern =>	/(^\d+$)|(^[a-z]{1,3}$)/,
	}
  HTML = [ :html ]
end

class TestValidator < Test::Unit::TestCase
  def setup
    @val = Validator.new
  end
	def test_reset_errors
		assert_equal({}, @val.errors)
		@val.errors = {:foo=>:bar}
		assert_nothing_raised {
			@val.reset_errors
		}
		assert_equal({}, @val.errors)
	end
	def test_validate
		assert_nothing_raised { @val.validate(:something, :other) }
	end
	def test_boolean
		assert_equal(true, @val.validate(:bool, 'true'))
		assert_equal(true, @val.validate(:bool, 'TRUE'))
		assert_equal(true, @val.validate(:bool, 1))
		assert_equal(true, @val.validate(:bool, true))
		assert_equal(true, @val.validate(:bool, 'Y'))
		assert_equal(true, @val.validate(:bool, 'j'))
		assert_equal(false, @val.validate(:bool, 'false'))
		assert_equal(false, @val.validate(:bool, 'FALSE'))
		assert_equal(false, @val.validate(:bool, 0))
		assert_equal(false, @val.validate(:bool, false))
		assert_equal(false, @val.validate(:bool, 'N'))
	end
	def test_enum
		assert_equal('foo', @val.validate(:enum, 'foo'))
	end
	def test_event
		assert_equal(:logout, @val.validate(:event, 'logout'))
		assert_equal(:logout, @val.validate(:event, ['logout']))
	end
	def test_email1
		assert_equal('e_invalid_email_address', @val.validate(:email, 'test').message)
		assert_equal('<EMAIL>', @val.validate(:email, '<EMAIL>'))
		assert_equal('<EMAIL>', @val.validate(:email, '<EMAIL>'))
		assert_equal(SBSM::InvalidDataError, @val.validate(:email, 'test@test@<EMAIL>').class)
	end
	def test_pass
		assert_equal('<PASSWORD>', @val.validate(:pass, 'test'))
	end
	def test_valid_values
		expected = ['1', '2', '3', '4', '7']
		assert_equal(expected, @val.valid_values(:values))
		assert_equal(expected, @val.valid_values('values'))
	end
	def test_date
		assert_equal(Date.new(2002,1,2), @val.validate(:date, '2.1.2002'))
		assert_equal(SBSM::InvalidDataError, @val.validate(:date, '13.13.1234').class)
		assert_equal(nil, @val.validate(:date, " \t"))
	end
	def test_state_id
		assert_equal(nil, @val.validate(:state_id, nil))
		assert_equal(nil, @val.validate(:state_id, "df"))
		assert_equal(1245, @val.validate(:state_id, "1245"))
		assert_equal(-1245, @val.validate(:state_id, "-1245"))
	end
	def test_pattern
		assert_equal('new', @val.validate(:pattern, 'new'))
		assert_equal('12345', @val.validate(:pattern, '12345'))
		assert_equal(nil, @val.validate(:pattern, '23foo45'))
		assert_equal(nil, @val.validate(:pattern, 'abfoodc'))
	end
  def test_validate_html
    src = "<SPAN style=\"PADDING-BOTTOM: 4px; LINE-HEIGHT: 1.4em; WHITE-SPACE: normal\"><P class=MsoNormal style=\"MARGIN: 0cm -0.3pt 0pt 0cm; TEXT-ALIGN: justify\"><SPAN lang=DE style=\"FONT-SIZE: 11pt; FONT-FAMILY: Arial; mso-bidi-font-size: 10.0pt; mso-bidi-font-family: 'Times New Roman'\">Wirkstoff: Ibuprofenum. <?xml:namespace prefix = o ns = \"urn:schemas-microsoft-com:office:office\" /><o:p></o:p></SPAN></P><P class=MsoNormal style=\"MARGIN: 0cm -0.3pt 0pt 0cm; TEXT-ALIGN: justify\"><SPAN lang=DE style=\"FONT-SIZE: 11pt; FONT-FAMILY: Arial; mso-bidi-font-size: 10.0pt; mso-bidi-font-family: 'Times New Roman'\">Hilfsstoffe: Conserv.: Sorbins\344ure (E 200), Excipiens pro compr. obducto<o:p></o:p></SPAN></P><P style=\"MARGIN-TOP: 4px; FONT-SIZE: 13px; LINE-HEIGHT: 1.4em\"></SPAN><BR>&nbsp;</P>"
    expected = "<span style=\"PADDING-BOTTOM: 4px; LINE-HEIGHT: 1.4em; WHITE-SPACE: normal\"><p class=\"MsoNormal\" style=\"MARGIN: 0cm -0.3pt 0pt 0cm; TEXT-ALIGN: justify\"><span lang=\"DE\" style=\"FONT-SIZE: 11pt; FONT-FAMILY: Arial; mso-bidi-font-size: 10.0pt; mso-bidi-font-family: 'Times New Roman'\">Wirkstoff: Ibuprofenum. </span></p><p class=\"MsoNormal\" style=\"MARGIN: 0cm -0.3pt 0pt 0cm; TEXT-ALIGN: justify\"><span lang=\"DE\" style=\"FONT-SIZE: 11pt; FONT-FAMILY: Arial; mso-bidi-font-size: 10.0pt; mso-bidi-font-family: 'Times New Roman'\">Hilfsstoffe: Conserv.: Sorbins\344ure (E 200), Excipiens pro compr. obducto</span></p><p style=\"MARGIN-TOP: 4px; FONT-SIZE: 13px; LINE-HEIGHT: 1.4em\"></p></span><br />&nbsp;"
    assert_equal expected, @val.validate(:html, src)
  end
end
<commit_msg>Added a test for whitespace in HTML-Validation<commit_after>#!/usr/bin/env ruby
#
# State Based Session Management	
#	Copyright (C) 2004 <NAME>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#	ywesee - intellectual capital connected, Winterthurerstrasse 52, CH-8006 Zrich, Switzerland
#	<EMAIL>
#
# TestValidator -- sbsm -- 15.11.2002 -- <EMAIL> 

$: << File.dirname(__FILE__)
$: << File.expand_path("../lib", File.dirname(__FILE__))

require 'test/unit'
require 'sbsm/validator'

class Validator < SBSM::Validator
	attr_accessor :errors
	DATES = [
		:date
	]
	ENUMS = {
		:enum	=>	['foo'],
		:values =>	['1', '2', '3', '4', '7'],
	}
	EVENTS = [
		:logout,
	]
	BOOLEAN = [ :bool ]
	PATTERNS = {
		:pattern =>	/(^\d+$)|(^[a-z]{1,3}$)/,
	}
  HTML = [ :html ]
end

class TestValidator < Test::Unit::TestCase
  def setup
    @val = Validator.new
  end
	def test_reset_errors
		assert_equal({}, @val.errors)
		@val.errors = {:foo=>:bar}
		assert_nothing_raised {
			@val.reset_errors
		}
		assert_equal({}, @val.errors)
	end
	def test_validate
		assert_nothing_raised { @val.validate(:something, :other) }
	end
	def test_boolean
		assert_equal(true, @val.validate(:bool, 'true'))
		assert_equal(true, @val.validate(:bool, 'TRUE'))
		assert_equal(true, @val.validate(:bool, 1))
		assert_equal(true, @val.validate(:bool, true))
		assert_equal(true, @val.validate(:bool, 'Y'))
		assert_equal(true, @val.validate(:bool, 'j'))
		assert_equal(false, @val.validate(:bool, 'false'))
		assert_equal(false, @val.validate(:bool, 'FALSE'))
		assert_equal(false, @val.validate(:bool, 0))
		assert_equal(false, @val.validate(:bool, false))
		assert_equal(false, @val.validate(:bool, 'N'))
	end
	def test_enum
		assert_equal('foo', @val.validate(:enum, 'foo'))
	end
	def test_event
		assert_equal(:logout, @val.validate(:event, 'logout'))
		assert_equal(:logout, @val.validate(:event, ['logout']))
	end
	def test_email1
		assert_equal('e_invalid_email_address', @val.validate(:email, 'test').message)
		assert_equal('<EMAIL>', @val.validate(:email, '<EMAIL>'))
		assert_equal('<EMAIL>', @val.validate(:email, '<EMAIL>'))
		assert_equal(SBSM::InvalidDataError, @val.validate(:email, 'test@test@test').class)
	end
	def test_pass
		assert_equal('<PASSWORD>6<PASSWORD>', @val.validate(:pass, 'test'))
	end
	def test_valid_values
		expected = ['1', '2', '3', '4', '7']
		assert_equal(expected, @val.valid_values(:values))
		assert_equal(expected, @val.valid_values('values'))
	end
	def test_date
		assert_equal(Date.new(2002,1,2), @val.validate(:date, '2.1.2002'))
		assert_equal(SBSM::InvalidDataError, @val.validate(:date, '13.13.1234').class)
		assert_equal(nil, @val.validate(:date, " \t"))
	end
	def test_state_id
		assert_equal(nil, @val.validate(:state_id, nil))
		assert_equal(nil, @val.validate(:state_id, "df"))
		assert_equal(1245, @val.validate(:state_id, "1245"))
		assert_equal(-1245, @val.validate(:state_id, "-1245"))
	end
	def test_pattern
		assert_equal('new', @val.validate(:pattern, 'new'))
		assert_equal('12345', @val.validate(:pattern, '12345'))
		assert_equal(nil, @val.validate(:pattern, '23foo45'))
		assert_equal(nil, @val.validate(:pattern, 'abfoodc'))
	end
  def test_validate_html
    src = "<SPAN style=\"PADDING-BOTTOM: 4px; LINE-HEIGHT: 1.4em; WHITE-SPACE: normal\"><P class=MsoNormal style=\"MARGIN: 0cm -0.3pt 0pt 0cm; TEXT-ALIGN: justify\"><SPAN lang=DE style=\"FONT-SIZE: 11pt; FONT-FAMILY: Arial; mso-bidi-font-size: 10.0pt; mso-bidi-font-family: 'Times New Roman'\">Wirkstoff: Ibuprofenum. <?xml:namespace prefix = o ns = \"urn:schemas-microsoft-com:office:office\" /><o:p></o:p></SPAN></P><P class=MsoNormal style=\"MARGIN: 0cm -0.3pt 0pt 0cm; TEXT-ALIGN: justify\"><SPAN lang=DE style=\"FONT-SIZE: 11pt; FONT-FAMILY: Arial; mso-bidi-font-size: 10.0pt; mso-bidi-font-family: 'Times New Roman'\">Hilfsstoffe: Conserv.: Sorbins\344ure (E 200), Excipiens pro compr. obducto<o:p></o:p></SPAN></P><P style=\"MARGIN-TOP: 4px; FONT-SIZE: 13px; LINE-HEIGHT: 1.4em\"></SPAN><BR>&nbsp;</P>"
    expected = "<span style=\"PADDING-BOTTOM: 4px; LINE-HEIGHT: 1.4em; WHITE-SPACE: normal\"><p class=\"MsoNormal\" style=\"MARGIN: 0cm -0.3pt 0pt 0cm; TEXT-ALIGN: justify\"><span lang=\"DE\" style=\"FONT-SIZE: 11pt; FONT-FAMILY: Arial; mso-bidi-font-size: 10.0pt; mso-bidi-font-family: 'Times New Roman'\">Wirkstoff: Ibuprofenum. </span></p><p class=\"MsoNormal\" style=\"MARGIN: 0cm -0.3pt 0pt 0cm; TEXT-ALIGN: justify\"><span lang=\"DE\" style=\"FONT-SIZE: 11pt; FONT-FAMILY: Arial; mso-bidi-font-size: 10.0pt; mso-bidi-font-family: 'Times New Roman'\">Hilfsstoffe: Conserv.: Sorbins\344ure (E 200), Excipiens pro compr. obducto</span></p><p style=\"MARGIN-TOP: 4px; FONT-SIZE: 13px; LINE-HEIGHT: 1.4em\"></p></span><br />&nbsp;"
    assert_equal expected, @val.validate(:html, src)
  end
  def test_validate_html__pre
    src = "<pre>     fooo     </pre>"
    expected = "<pre>     fooo     </pre>"
    assert_equal expected, @val.validate(:html, src)
  end
end

{'fraction_non_alphanumeric': 0.17845209067100154, 'fraction_numerical': 0.030253770432583763, 'mean_word_length': 5.005422993492408, 'pattern_counts': {'":': 0, '<': 99, '<?xml version=': 0, '>': 95, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '624435', 'n_tokens_mistral': 4417, 'n_tokens_neox': 4060, 'n_words': 947}