<commit_before>package rocket


type Program string

func lex (program Program) []Unit {

	currentIndex := 0;
	unitList := []Unit{}

	for currentIndex < len(program) {
		unit := string(program[currentIndex])
		unitType := determineType(unit)
		switch unitType {
			case "INTERGER":
				nextType := determineType(program.peek(currentIndex+1))
				if nextType == "INTERGER" {
					completeUnit := program.peekLong(currentIndex, "INTERGER")
					unitList = append(unitList, completeUnit)
					currentIndex += len(completeUnit.cargo)			
				} else {
					unitList = append(unitList, Digits.getBaseUnit(unit))
					currentIndex++
				}
			case "LETTER": //not needed for now
			case "OPERATOR":
				nextUnit := program.peek(currentIndex+1)
				nextType := determineType(program.peek(currentIndex+1))
				if nextType == "OPERATOR" {
					newOperator := unit+nextUnit
					definition := TwoCharacterSymbols.getBaseUnit(newOperator)
					if  definition.cargo != "BASE_LOOKUP_ERR" {
						unitList = append(unitList,definition)
						currentIndex += 2
					} else {
						unitList = append(unitList,OneCharacterSymbols.getBaseUnit(unit))
						currentIndex ++
					}
				} else {
					unitList = append(unitList,OneCharacterSymbols.getBaseUnit(unit))
					currentIndex++
				}
			case "SYMBOL":
				unitList = append(unitList, OneCharacterSymbols.getBaseUnit(unit))
				currentIndex++
		}
	}
	return unitList
}

func (program Program) peek (index int) string {
	return string(program[index])
}

func (program Program) peekLong (startIndex int, unitType string) Unit {
	currentIndex := startIndex
	accum := ""
	for ;; {
		if(currentIndex >= len(program)) {
			return Unit{cargo: accum, notation: unitType, unitType: unitType}
			break
		}
		currentUnit := string(program[currentIndex])
		if determineType(currentUnit) != unitType {
			return Unit{cargo: accum, notation: unitType, unitType: unitType}
			break
		} else {
			accum += currentUnit
			currentIndex++
		}
	}

	return Unit{cargo: "PEEKLONG_ERR", notation: "PEEKLONG_ERR", unitType: "INTERGER"}
}

func determineType (unit string) string {
	if Digits.contains(unit) {
		return "INTERGER"
	}
	if Letters.contains(unit) {
		return "LETTER"
	}
	if OneCharacterSymbols.contains(unit) {
		return OneCharacterSymbols.getType(unit)
	}
	return "UNDEFINED_TYPE"
}

func (unitTable UnitTable) contains (currentUnit string) bool {
    for _, tableUnit := range unitTable {
        if tableUnit.cargo == currentUnit {
            return true
        }
    }
    return false
}

func (unitTable UnitTable) getBaseUnit (currentUnit string) Unit {
	for _, tableUnit := range unitTable {
		if tableUnit.cargo == currentUnit {
            return tableUnit
        }
	}
	return Unit{cargo: "BASE_LOOKUP_ERR", notation: "BASE_LOOKUP_ERR", unitType: "BASE_LOOKUP_ERR"}
}

func (unitTable UnitTable) getType (currentUnit string) string {
	for _, tableUnit := range unitTable {
		if tableUnit.cargo == currentUnit {
            return tableUnit.unitType
        }
	}
	return "UNKNOWN_TYPE"
}<commit_msg>Fix dangerous fallthrough of unneeded case<commit_after>package rocket


type Program string

func lex (program Program) []Unit {

	currentIndex := 0;
	unitList := []Unit{}

	for currentIndex < len(program) {
		unit := string(program[currentIndex])
		unitType := determineType(unit)
		switch unitType {
			case "INTERGER":
				nextType := determineType(program.peek(currentIndex+1))
				if nextType == "INTERGER" {
					completeUnit := program.peekLong(currentIndex, "INTERGER")
					unitList = append(unitList, completeUnit)
					currentIndex += len(completeUnit.cargo)			
				} else {
					unitList = append(unitList, Digits.getBaseUnit(unit))
					currentIndex++
				}
			case "LETTER": //not needed for now
				break;
			case "OPERATOR":
				nextUnit := program.peek(currentIndex+1)
				nextType := determineType(program.peek(currentIndex+1))
				if nextType == "OPERATOR" {
					newOperator := unit+nextUnit
					definition := TwoCharacterSymbols.getBaseUnit(newOperator)
					if  definition.cargo != "BASE_LOOKUP_ERR" {
						unitList = append(unitList,definition)
						currentIndex += 2
					} else {
						unitList = append(unitList,OneCharacterSymbols.getBaseUnit(unit))
						currentIndex ++
					}
				} else {
					unitList = append(unitList,OneCharacterSymbols.getBaseUnit(unit))
					currentIndex++
				}
			case "SYMBOL":
				unitList = append(unitList, OneCharacterSymbols.getBaseUnit(unit))
				currentIndex++
		}
	}
	return unitList
}

func (program Program) peek (index int) string {
	return string(program[index])
}

func (program Program) peekLong (startIndex int, unitType string) Unit {
	currentIndex := startIndex
	accum := ""
	for ;; {
		if(currentIndex >= len(program)) {
			return Unit{cargo: accum, notation: unitType, unitType: unitType}
			break
		}
		currentUnit := string(program[currentIndex])
		if determineType(currentUnit) != unitType {
			return Unit{cargo: accum, notation: unitType, unitType: unitType}
			break
		} else {
			accum += currentUnit
			currentIndex++
		}
	}

	return Unit{cargo: "PEEKLONG_ERR", notation: "PEEKLONG_ERR", unitType: "INTERGER"}
}

func determineType (unit string) string {
	if Digits.contains(unit) {
		return "INTERGER"
	}
	if Letters.contains(unit) {
		return "LETTER"
	}
	if OneCharacterSymbols.contains(unit) {
		return OneCharacterSymbols.getType(unit)
	}
	return "UNDEFINED_TYPE"
}

func (unitTable UnitTable) contains (currentUnit string) bool {
    for _, tableUnit := range unitTable {
        if tableUnit.cargo == currentUnit {
            return true
        }
    }
    return false
}

func (unitTable UnitTable) getBaseUnit (currentUnit string) Unit {
	for _, tableUnit := range unitTable {
		if tableUnit.cargo == currentUnit {
            return tableUnit
        }
	}
	return Unit{cargo: "BASE_LOOKUP_ERR", notation: "BASE_LOOKUP_ERR", unitType: "BASE_LOOKUP_ERR"}
}

func (unitTable UnitTable) getType (currentUnit string) string {
	for _, tableUnit := range unitTable {
		if tableUnit.cargo == currentUnit {
            return tableUnit.unitType
        }
	}
	return "UNKNOWN_TYPE"
}

{'fraction_non_alphanumeric': 0.10994592823201704, 'fraction_numerical': 0.0016385384237260364, 'mean_word_length': 3.7026194144838214, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3642242', 'n_tokens_mistral': 2213, 'n_tokens_neox': 1909, 'n_words': 444}