<commit_before>package ilang

import "strconv"
import "strings"
//import "github.com/gedex/inflector"

func (ic *Compiler) expression() string {
	var token = ic.Scan(0)
	
	for token == "\n" {
		token = ic.Scan(0)
	}
	
	switch token {
		case "true":
			ic.ExpressionType = Number
			return "1"
		case "false":
			ic.ExpressionType = Number
			return "0"
		case "error":
			ic.ExpressionType = Number
			return "ERROR"
	}
	
	//Text.
	if token[0] == '"' || token[0] == '`' {
		ic.ExpressionType = Text
		return ic.ParseString(token)
	}
	
	//Hexadecimal.
	if len(token) > 2 && token[0] == '0' && token[1] == 'x' { 
		ic.ExpressionType = Number
		return token
	}
	
	//Subexpessions.
	if token == "(" {
		defer func() {
			ic.Scan(')')
		}()
		return ic.ScanExpression()
	}
	
	//Is it a literal number? Then just return it.
	if _, err := strconv.Atoi(token); err == nil{
		ic.ExpressionType = Number
		return token
	}
	
	//Minus.
	if token == "-" {
		ic.NextToken = token
		ic.ExpressionType = Number
		return ic.Shunt("0")
	}
	
	if t := ic.GetVariable(token); t != Undefined {
	
		ic.ExpressionType = t
		ic.SetVariable(token+"_use", Used)
		
		return token
	}
	
	if ic.TypeExists(token) {
		ic.ExpressionType = ic.DefinedTypes[token]
		
		//This is a constructor. eg. var bug = Bug(); where Bug is a type.
		if ic.Peek() == "(" || ic.NextToken == "(" {
			ic.Scan('(')
			ic.Scan(')')
			
			return ic.CallType(token)
			
		//This is a type literal.
		} else if ic.Peek() == "{" {
			ic.NextToken = token
			variable := ic.ScanTypeLiteral()
				//TODO better gc protection.
			ic.SetVariable(variable, ic.DefinedTypes[token])
			ic.SetVariable(variable+"_use", Used)<commit_msg>Indexing needs to take precident over anything else.<commit_after>package ilang

import "strconv"
import "strings"
//import "github.com/gedex/inflector"

func (ic *Compiler) expression() string {
	var token = ic.Scan(0)
	
	for token == "\n" {
		token = ic.Scan(0)
	}
	
	switch token {
		case "true":
			ic.ExpressionType = Number
			return "1"
		case "false":
			ic.ExpressionType = Number
			return "0"
		case "error":
			ic.ExpressionType = Number
			return "ERROR"
	}
	
	//Text.
	if token[0] == '"' || token[0] == '`' {
		ic.ExpressionType = Text
		return ic.ParseString(token)
	}
	
	//Hexadecimal.
	if len(token) > 2 && token[0] == '0' && token[1] == 'x' { 
		ic.ExpressionType = Number
		return token
	}
	
	//Subexpessions.
	if token == "(" {
		defer func() {
			ic.Scan(')')
		}()
		return ic.ScanExpression()
	}
	
	//Is it a literal number? Then just return it.
	if _, err := strconv.Atoi(token); err == nil{
		ic.ExpressionType = Number
		return token
	}
	
	//Minus.
	if token == "-" {
		ic.NextToken = token
		ic.ExpressionType = Number
		return ic.Shunt("0")
	}
	
	if t := ic.GetVariable(token); t != Undefined {
	
		ic.ExpressionType = t
		ic.SetVariable(token+"_use", Used)
		
		if ic.Peek() == "." {
			return ic.Shunt(token)
		}
		
		return token
	}
	
	if ic.TypeExists(token) {
		ic.ExpressionType = ic.DefinedTypes[token]
		
		//This is a constructor. eg. var bug = Bug(); where Bug is a type.
		if ic.Peek() == "(" || ic.NextToken == "(" {
			ic.Scan('(')
			ic.Scan(')')
			
			return ic.CallType(token)
			
		//This is a type literal.
		} else if ic.Peek() == "{" {
			ic.NextToken = token
			variable := ic.ScanTypeLiteral()
				//TODO better gc protection.
			ic.SetVariable(variable, ic.DefinedTypes[token])
			ic.SetVariable(variable+"_use", Used)
{'fraction_non_alphanumeric': 0.16492450638792103, 'fraction_numerical': 0.006387921022067364, 'mean_word_length': 2.8448660714285716, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4507878', 'n_tokens_mistral': 1430, 'n_tokens_neox': 1246, 'n_words': 312}