<commit_before>require 'spec_helper'

describe Coke::Lexer do

	let(:lexer) { Coke::Lexer.new }
	def lexed code
		lexer.tokenize code
	end
	def lexedf filename
    open(File.expand_path(File.join(__FILE__, "..", "..", "fixtures", filename)), "r") do |f|
      lexer.tokenize f.read
    end
	end

	# REX LEXING
	# def lex str
	# 	@scanner = Coke::Parser.new
	# 	@scanner.scan_setup str
	# end

	# def lexed(scanner = @scanner)
	# 	tokens = []
	# 	while token = scanner.next_token
	# 		tokens << token
	# 	end
	# 	tokens
	# end

	describe "helloworld.coke" do

		let(:l){lexedf "helloworld.coke"}

		it "lexes right" do
			l.should == [
				[:COMMENT, " Comment"],
				[:NEWLINE, "\n"],
				[:IDENTIFIER, "log"],
				[:STRING, "hello world"]
			]
		end

	end

	it "lexes spaces to whitespace" do
		lexed("  identifier").should == [[:IDENTIFIER, "identifier"]]
	end

	it "lexes tabs to whitespace" do
		# lexed("\t\tidentifier").should == [[:IDENTIFIER, "identifier"]]
	end

	describe "lexes method calls" do

		it "lexes methods with parentheses" do
			lexed("NSBundle.mainBundle()").should == [
				[:CONSTANT, "NSBundle"],
				[".", "."],
				[:IDENTIFIER, "mainBundle"],
				["(", "("],
				[")", ")"]
			]
		end

		it "lexes methods with a colon in the name" do
			lexed("NSBundle.mainBundle:()").should == [
				[:CONSTANT, "NSBundle"],
				[".", "."],
				[:IDENTIFIER, "mainBundle"],
				[":", ":"],
				["(", "("],
				[")", ")"]
			]
		end
<commit_msg>bumpus<commit_after>require 'spec_helper'

describe Coke::Lexer do

	let(:lexer) { Coke::Lexer.new }
	def lexed code
		lexer.tokenize code
	end
	def lexedf filename
    open(File.expand_path(File.join(__FILE__, "..", "..", "fixtures", filename)), "r") do |f|
      lexer.tokenize f.read
    end
	end

	# REX LEXING
	# def lex str
	# 	@scanner = Coke::Parser.new
	# 	@scanner.scan_setup str
	# end

	# def lexed(scanner = @scanner)
	# 	tokens = []
	# 	while token = scanner.next_token
	# 		tokens << token
	# 	end
	# 	tokens
	# end

	describe "helloworld.coke" do

		let(:l){lexedf "helloworld.coke"}

		it "lexes right" do
			l.should == [
				[:COMMENT, " Comment"],
				[:NEWLINE, "\n"],
				[:IDENTIFIER, "log"],
				[:STRING, "hello world"]
			]
		end

	end

	it "lexes spaces to whitespace" do
		lexed("  identifier").should == [[:IDENTIFIER, "identifier"]]
	end

	it "lexes tabs to whitespace" do
		# # lexed("\t\tidentifier").should == [[:IDENTIFIER, "identifier"]]
	end

	describe "lexes method calls" do

		it "lexes methods with parentheses" do
			lexed("NSBundle.mainBundle()").should == [
				[:CONSTANT, "NSBundle"],
				[".", "."],
				[:IDENTIFIER, "mainBundle"],
				["(", "("],
				[")", ")"]
			]
		end

		it "lexes methods with a colon in the name" do
			lexed("NSBundle.mainBundle:()").should == [
				[:CONSTANT, "NSBundle"],
				[".", "."],
				[:IDENTIFIER, "mainBundle"],
				[":", ":"],
				["(", "("],
				[")", ")"]
			]
		end

{'fraction_non_alphanumeric': 0.1960178510127017, 'fraction_numerical': 0.0, 'mean_word_length': 3.047222222222222, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5413616', 'n_tokens_mistral': 1304, 'n_tokens_neox': 1113, 'n_words': 246}