<commit_before><?php
	use PHPUnit\Framework\TestCase;

	final class LexTest extends TestCase{

		private function lex_test($str, $tokens){
			$obj = new iamcal\SQLParser();

			$this->assertEquals($obj->lex($str), $tokens);
		}

		public function test_simple_word_tokens(){

			$this->lex_test('hello world', array('hello', 'world'));
		}

		public function testWhitespace(){

			$this->lex_test("foo bar", array("foo", "bar"));
			$this->lex_test(" foo bar ", array("foo", "bar"));
			$this->lex_test("\n\nfoo\tbar\n", array("foo", "bar"));

			# with comments too
			$this->lex_test("hello \nworld-- foo\nyeah", array('hello', 'world', 'yeah'));
		}

		public function testComments(){
			$this->lex_test("foo -- bar", array("foo"));
			$this->lex_test("foo --bar", array("foo"));
			$this->lex_test("foo -- bar\n", array("foo"));
			$this->lex_test("foo -- bar \n", array("foo"));
			$this->lex_test("foo -- bar \n ", array("foo"));


			$this->lex_test("foo/"."* hello *"."/ bar", array("foo", "bar"));
			$this->lex_test("foo/"."*hello*"."/ bar", array("foo", "bar"));
			$this->lex_test("foo/"."* hello \n world *"."/ bar", array("foo", "bar"));
			$this->lex_test("foo/"."*hello \n world*"."/ bar", array("foo", "bar"));
		}

		public function testBacktickFields(){

			$this->lex_test("hello `world` foo", array("hello", "`world`", "foo"));<commit_msg>added lexer tests for multiline comments that don't end<commit_after><?php
	use PHPUnit\Framework\TestCase;

	final class LexTest extends TestCase{

		private function lex_test($str, $tokens){
			$obj = new iamcal\SQLParser();

			$this->assertEquals($obj->lex($str), $tokens);
		}

		public function test_simple_word_tokens(){

			$this->lex_test('hello world', array('hello', 'world'));
		}

		public function testWhitespace(){

			$this->lex_test("foo bar", array("foo", "bar"));
			$this->lex_test(" foo bar ", array("foo", "bar"));
			$this->lex_test("\n\nfoo\tbar\n", array("foo", "bar"));

			# with comments too
			$this->lex_test("hello \nworld-- foo\nyeah", array('hello', 'world', 'yeah'));
		}

		public function testComments(){
			$this->lex_test("foo -- bar", array("foo"));
			$this->lex_test("foo --bar", array("foo"));
			$this->lex_test("foo -- bar\n", array("foo"));
			$this->lex_test("foo -- bar \n", array("foo"));
			$this->lex_test("foo -- bar \n ", array("foo"));

			$this->lex_test("foo/"."* hello *"."/ bar", array("foo", "bar"));
			$this->lex_test("foo/"."*hello*"."/ bar", array("foo", "bar"));
			$this->lex_test("foo/"."* hello \n world *"."/ bar", array("foo", "bar"));
			$this->lex_test("foo/"."*hello \n world*"."/ bar", array("foo", "bar"));

			$this->lex_test("foo/"."* hello", array("foo"));
			$this->lex_test("foo/"."* hello *", array("foo"));
			$this->lex_test("foo/"."* hello \n world", array("foo"));
		}

		public function testBacktickFields(){

			$this->lex_test("hello `world` foo", array("hello", "`world`", "foo"));
{'fraction_non_alphanumeric': 0.2586265801161599, 'fraction_numerical': 0.0, 'mean_word_length': 4.266187050359712, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7441456', 'n_tokens_mistral': 1143, 'n_tokens_neox': 1007, 'n_words': 216}