<commit_before>import Foundation

class Scanner: KeywordProvider {
	
	enum ScannerState {
		case IdleState // just waiting
		case InitialState
		case LiteralState
		case IdentState
		case SymbolState
		case ErrorState(description: String)
	}
	
	struct Line {
		var number: Int = 0
		var iterator: IndexingGenerator<String.CharacterView>
		var content: String = "" {
			didSet {
				// rebuild the iterator when the content changes
				self.iterator = self.content.characters.generate()
			}
		}
		
		init(content: String, number: Int) {
			self.content = content
			self.number = number
			self.iterator = self.content.characters.generate()
		}
	}
	
	// the current token range "under construction" once an entire
	// token is found this is reset to its initial value (0, 0)
	var currentTokenRange: Range = Range(start: 0, end: 0)
	
	// the current line (we read the file line wise)
	var currentLine: Line = Line(content: "", number: 0)
	
	// the current state: as long as it is in remains in .IdleState
	// it will not do anything. Once we have transition <to> any other
	// state it will start processing currentLine till it will
	// eventually end up in .IdleState again.
	var currentState: ScannerState = .IdleState {
		didSet {
			if currentLine.content != "" {
				stateChangeHandler(from: oldValue, to: currentState)
			}
		}
	}
	
	// the most important thing here
	var tokenlist: [Token] = []
	
	///////////////////////////////////////////////////////////
	// Scanner Scan Functions
	///////////////////////////////////////////////////////////
	
	// Reads the file line wise and returns the tokenlist
	// for the token list
	func scan (fromPath: String) -> [Token]? {
		tokenlist = [] // reset old tokenlist
		if let lines = getContentByLine(fromPath) {
			for line in lines {
				prepareNewLine(line)
				processNewLine()
			}
		}
		
		print("\n✅ Scan finished: Tokenlist is:")
		for token in tokenlist {
			print("🔘\(token.terminal)\n➡️\(token.attribute)")
		}
		
		return tokenlist
	}
	
	///////////////////////////////////////////////////////////
	// Scanner Token Functions
	///////////////////////////////////////////////////////////
	
	private func buildTokenFrom(aRange: Range<Int>) -> String {
		// end position must be given as negative advanceBy(-n)!
		let fromTokenEnd = currentLine.content.characters.count - aRange.endIndex
		let toTokenStart = aRange.startIndex
		
		let tokenRange: Range<String.Index> = Range(
			start: currentLine.content.startIndex.advancedBy(toTokenStart),
			end: currentLine.content.endIndex.advancedBy(-fromTokenEnd))
		
		return currentLine.content.substringWithRange(tokenRange)
	}
	
	private func newLiteralToken() {
		let literal = buildTokenFrom(currentTokenRange)
        
        //check if Int is not out of Int64 range
        let checkInt = Double.init(literal)
        if checkInt != nil && checkInt > Double.init(integerLiteral: INT64_MAX) {
            currentState = .ErrorState(description: "❌ Int64 out of Range: \(currentLine.number)")
        } else {
            let token = Token(
                terminal: Terminal.LITERAL,
                lineNumber: currentLine.number,
                attribute: Token.Attribute.Integer(Int(literal)!))

            print("✅ New literal token: \(literal)")
            tokenlist.append(token)
		
            // be ready for the next token
            currentTokenRange.startIndex = currentTokenRange.endIndex
        }
	}
	
	private func newIdentifierToken() {
		let identifier = buildTokenFrom(currentTokenRange)
		
		if var keywordToken = matchKeyword(identifier) {
			keywordToken.lineNumber = currentLine.number;
			print("✅ New keyword token: \(identifier)")
			tokenlist.append(keywordToken)
		} else {
			let token = Token(
				terminal: Terminal.IDENT,
				lineNumber: currentLine.number,
				attribute: Token.Attribute.Ident(identifier))
			print("✅ New identifier token: \(identifier)")
			tokenlist.append(token)
		}
	}
	
	private func newSymbolToken() {
		let symbol = buildTokenFrom(currentTokenRange)
		
		if var keywordToken = matchKeyword(symbol) {
			keywordToken.lineNumber = currentLine.number;
			print("✅ New keyword token: \(symbol)")
			tokenlist.append(keywordToken)
		} else {
			currentState = .ErrorState(description: "❌ Unrecogizable symbol \(symbol)" )
		}
	}
	
	
	///////////////////////////////////////////////////////////
	// Scanner State Handling Functions
	///////////////////////////////////////////////////////////
	
	private func stateChangeHandler(from from: ScannerState, to: ScannerState) {
		print("➡️ Changed current state from <\(from)> to <\(to)>.")
		
		// we have reached the end of a token and come back
		// to the .InitialState => create the new token. :)
		switch (from, to) {
		case (.SymbolState, .SymbolState): fallthrough
		case (.IdentState, .IdentState): fallthrough
		case (.LiteralState, .LiteralState): break
		case (.LiteralState, _): newLiteralToken()
		case (.IdentState, _): newIdentifierToken()
		case (.SymbolState, _): newSymbolToken()
		case (_, .ErrorState): break // TODO: Fixme -> Throw exception or so.
		case (_, _): break
		}
		
		// update the end index for 
		// the next transition ahead<commit_msg>Comments improved<commit_after>import Foundation

class Scanner: KeywordProvider {
	
	enum ScannerState {
		case IdleState // just waiting
		case InitialState
		case LiteralState
		case IdentState
		case SymbolState
		case ErrorState(description: String)
	}
	
	struct Line {
		var number: Int = 0
		var iterator: IndexingGenerator<String.CharacterView>
		var content: String = "" {
			didSet {
				// rebuild the iterator when the content changes
				self.iterator = self.content.characters.generate()
			}
		}
		
		init(content: String, number: Int) {
			self.content = content
			self.number = number
			self.iterator = self.content.characters.generate()
		}
	}
	
	// the current token range "under construction" once an entire
	// token is found this is reset to its initial value (0, 0)
	var currentTokenRange: Range = Range(start: 0, end: 0)
	
	// the current line (we read the file line wise)
	var currentLine: Line = Line(content: "", number: 0)
	
	// the current state: as long as it is in remains in .IdleState
	// it will not do anything. Once we have transition <to> any other
	// state it will start processing currentLine till it will
	// eventually end up in .IdleState again.
	var currentState: ScannerState = .IdleState {
		didSet {
			if currentLine.content != "" {
				stateChangeHandler(from: oldValue, to: currentState)
			}
		}
	}
	
	// the most important thing here
	var tokenlist: [Token] = []
	
	///////////////////////////////////////////////////////////
	// Scanner Scan Functions
	///////////////////////////////////////////////////////////
	
	// Reads the file line wise and returns the tokenlist
	// for the token list
	func scan (fromPath: String) -> [Token]? {
		tokenlist = [] // reset old tokenlist
		if let lines = getContentByLine(fromPath) {
			for line in lines {
				prepareNewLine(line)
				processNewLine()
			}
		}
		
		print("\n✅ Scan finished: Tokenlist is:")
		for token in tokenlist {
			print("🔘\(token.terminal)\n➡️\(token.attribute)")
		}
		
		return tokenlist
	}
	
	///////////////////////////////////////////////////////////
	// Scanner Token Functions
	///////////////////////////////////////////////////////////
	
	private func buildTokenFrom(aRange: Range<Int>) -> String {
		// end position must be given as negative advanceBy(-n)!
		let fromTokenEnd = currentLine.content.characters.count - aRange.endIndex
		let toTokenStart = aRange.startIndex
		
		let tokenRange: Range<String.Index> = Range(
			start: currentLine.content.startIndex.advancedBy(toTokenStart),
			end: currentLine.content.endIndex.advancedBy(-fromTokenEnd))
		
		return currentLine.content.substringWithRange(tokenRange)
	}
	
	private func newLiteralToken() {
		let literal = buildTokenFrom(currentTokenRange)
        
        //check if Int is not out of Int64 range
        let checkInt = Double.init(literal)
        if checkInt != nil && checkInt > Double.init(integerLiteral: INT64_MAX) {
            currentState = .ErrorState(description: "❌ Int64 out of Range: \(currentLine.number)")
        } else {
            let token = Token(
                terminal: Terminal.LITERAL,
                lineNumber: currentLine.number,
                attribute: Token.Attribute.Integer(Int(literal)!))

            print("✅ New literal token: \(literal)")
            tokenlist.append(token)
		
            // be ready for the next token
            currentTokenRange.startIndex = currentTokenRange.endIndex
        }
	}
	
	private func newIdentifierToken() {
		let identifier = buildTokenFrom(currentTokenRange)
		
		if var keywordToken = matchKeyword(identifier) {
			keywordToken.lineNumber = currentLine.number;
			print("✅ New keyword token: \(identifier)")
			tokenlist.append(keywordToken)
		} else {
			let token = Token(
				terminal: Terminal.IDENT,
				lineNumber: currentLine.number,
				attribute: Token.Attribute.Ident(identifier))
			print("✅ New identifier token: \(identifier)")
			tokenlist.append(token)
		}
	}
	
	private func newSymbolToken() {
		let symbol = buildTokenFrom(currentTokenRange)
		
		if var keywordToken = matchKeyword(symbol) {
			keywordToken.lineNumber = currentLine.number;
			print("✅ New keyword token: \(symbol)")
			tokenlist.append(keywordToken)
		} else {
			currentState = .ErrorState(description: "❌ Unrecogizable symbol \(symbol)" )
		}
	}
	
	
	///////////////////////////////////////////////////////////
	// Scanner State Handling Functions
	///////////////////////////////////////////////////////////
	
	private func stateChangeHandler(from from: ScannerState, to: ScannerState) {
		print("➡️ Changed current state from <\(from)> to <\(to)>.")
		
		// we have reached the end of a token and come back
		// to the .InitialState => create the new token. :)
		switch (from, to) {
		case (.SymbolState, .SymbolState): fallthrough // continues
		case (.IdentState, .IdentState): fallthrough // continues
		case (.LiteralState, .LiteralState): break // continues
		case (.LiteralState, _): newLiteralToken()
		case (.IdentState, _): newIdentifierToken()
		case (.SymbolState, _): newSymbolToken()
		case (_, .ErrorState): break // TODO: Fixme -> Throw exception or so.
		case (_, _): break
		}
		
		// update the end index for 
		// the next transition ahead
{'fraction_non_alphanumeric': 0.16511292647765496, 'fraction_numerical': 0.0023065833733781835, 'mean_word_length': 2.4118032786885246, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2622622', 'n_tokens_mistral': 3213, 'n_tokens_neox': 2817, 'n_words': 865}