<commit_before>package stealer

import (
	"bytes"
	"errors"
	"os"
	"strings"
	"sync"
	"unicode"
)

// Read file and return its content.
func ReadFile(filepath string) (error, map[string][]string) {
	var result map[string][]string

	file, err := os.OpenFile(filepath, os.O_RDONLY, os.ModePerm)
	if err != nil {
		return err, nil
	}

	fileInfo, err := file.Stat()
	size := fileInfo.Size()
	datas := make([]byte, size)

	n, err := file.Read(datas)
	if err != nil {
		return err, nil
	}

	if n == 0 {
		err = errors.New("Empty selected file")
		return err, nil
	}

	result = getVariablesValues(datas)

	return nil, result
}

// Getting all the variables values that has prefix (protected,private,public)
// Return map with the key for variable name and []string for all the values from the variable
func getVariablesValues(datas []byte) map[string][]string {
	result := make(map[string][]string)
	accessKeys := []string{"private", "protected", "public"}
	for _, access := range accessKeys {
		datas := findData(access, datas)

		if len(datas) > 0 {
			// adding map datas to result
			for key, value := range datas {
				result[key] = value
			}
		}
	}

	return result
}

// Get all the values from a variable with specific access from the data.
// This acces can be private, protected or public.
func findData(access string, data []byte) map[string][]string {
	result := make(map[string][]string)
	function := []byte(`function`)
	lenAccess := len(access)

	for i := 0; i < len(data); {
		varIndex := bytes.Index(data[i:], []byte(access))
		if varIndex > -1 {
			i = i + varIndex + lenAccess
			// check if it is a variable or function
			indexEnter := bytes.Index(data[i:], []byte("\n"))
			funcIndex := bytes.Index(data[i:i+indexEnter], function)
			if funcIndex > -1 {
				// jump to specific index and continue search for variable
				i = i + indexEnter
				continue
			} else {
				// found a variable get variable name
				dollar := []byte(`$`)
				space := []byte(`=`)
				idxDollar := bytes.Index(data[i:], dollar)
				i = i + idxDollar
				idxSpace := bytes.Index(data[i:], space)
				varNameByte := data[i+1 : i+idxSpace]
				varName := removeSpace(string(varNameByte))

				//check if it is an array
				idxArray := bytes.Index(data[i:], []byte(`array`))
				if idxArray > -1 {
					i = i + idxArray
					// looop to get all the array
					varEndIndex := bytes.Index(data[i:], []byte(`;`))
					varEndIndex = i + varEndIndex
					index, values := getValues(i, varEndIndex, data)

					if len(values) > 0 {
						// found values insert to our map
						result[varName] = values
						i = index
					} else {
						// we did'nt found any values from the variable, skip it
						i = varEndIndex
					}

				}
			}

		} else {
			break
		}

	}
	return result
}

// Getting all values number from a variable.
// the number could be any data type like eg: int,float
// call this function if we got null values from getValuesString()
// Note i is current index in integer
func getValuesNumber(i, idxSemiColon int, data []byte) (int, []string) {
	var valuesNumber []string
	comma := []byte(`,`)
	oP := []byte(`(`) // oP -> open Parenthesis
	cP := []byte(`)`) // cP ->close Parenthesis
	var cPindex int
	var oPindex int
	var commaIndex int

	// finding the open parenthesis `(` character and assign tu current index = i
	oPindex = findChar(data, oP, i, idxSemiColon)

	// check oPindex
	if oPindex > -1 {
		i = oPindex + 1
	}

	if oPindex >= idxSemiColon || oPindex == -1 {
		return i, valuesNumber
	}

	for {
		commaIndex = findChar(data, comma, i, idxSemiColon)
		if commaIndex >= idxSemiColon {
			break
		}
		if commaIndex > -1 {
			// found separator get the value
			// set and check separatorIndex
			addValueStore(&valuesNumber, data, i, commaIndex)
			i = commaIndex + 1

			if i >= idxSemiColon {
				break
			}
		} else {
			// find close Parenthesis
			cPindex = findChar(data, cP, i, idxSemiColon)
			if cPindex >= idxSemiColon || cPindex == -1 {
				break
			}

			addValueStore(&valuesNumber, data, i, cPindex)
			i = cPindex + 1

		}

	}

	return i, valuesNumber
}

// Adding value to store from given data within the currentIndex and lastIndex
// remove the whitespaces if exist
// and return the currentIndex which modified from this function
func addValueStore(store *[]string, data []byte, currentIndex int, lastIndex int) {
	if currentIndex < lastIndex && lastIndex <= len(data) {
		value := data[currentIndex:lastIndex]
		if string(value) != "" {
			valueString := strings.TrimSpace(string(value))
			*store = append(*store, valueString)
		}
	}
}

// find the char index inside data from the given of currentIndex
// This currentIndex is to avoid getting previous charIndex
func findChar(data []byte, char []byte, currentIndex, idxSemiColon int) int {
	charIndex := -1
	if currentIndex < idxSemiColon && currentIndex < len(data) && idxSemiColon < len(data) {
		charIndex = bytes.Index(data[currentIndex:idxSemiColon], char)
		if charIndex > -1 {
			//found char  inside data
			charIndex = currentIndex + charIndex
		}
	}

	return charIndex
}

// Getting the values from data with index i to idxSemiColon
// In php you can use single "'" and double "'".
// This function will get the values concurrently for the string values
// if string values not exist then it will try to get the numbers
func getValues(i, idxSemiColon int, data []byte) (int, []string) {
	// type and variable to hold the result
	type storage struct {
		Index  int
		Values []string
	}<commit_msg>removeSpace to remove all whitespaces faster way<commit_after>package stealer

import (
	"bytes"
	"errors"
	"os"
	"strings"
	"sync"
	"unicode"
)

// Read file and return its content.
func ReadFile(filepath string) (error, map[string][]string) {
	var result map[string][]string

	file, err := os.OpenFile(filepath, os.O_RDONLY, os.ModePerm)
	if err != nil {
		return err, nil
	}

	fileInfo, err := file.Stat()
	size := fileInfo.Size()
	datas := make([]byte, size)

	n, err := file.Read(datas)
	if err != nil {
		return err, nil
	}

	if n == 0 {
		err = errors.New("Empty selected file")
		return err, nil
	}

	result = getVariablesValues(datas)

	return nil, result
}

// Getting all the variables values that has prefix (protected,private,public)
// Return map with the key for variable name and []string for all the values from the variable
func getVariablesValues(datas []byte) map[string][]string {
	result := make(map[string][]string)
	accessKeys := []string{"private", "protected", "public"}
	for _, access := range accessKeys {
		datas := findData(access, datas)

		if len(datas) > 0 {
			// adding map datas to result
			for key, value := range datas {
				result[key] = value
			}
		}
	}

	return result
}

// Get all the values from a variable with specific access from the data.
// This acces can be private, protected or public.
func findData(access string, data []byte) map[string][]string {
	result := make(map[string][]string)
	function := []byte(`function`)
	lenAccess := len(access)

	for i := 0; i < len(data); {
		varIndex := bytes.Index(data[i:], []byte(access))
		if varIndex > -1 {
			i = i + varIndex + lenAccess
			// check if it is a variable or function
			indexEnter := bytes.Index(data[i:], []byte("\n"))
			funcIndex := bytes.Index(data[i:i+indexEnter], function)
			if funcIndex > -1 {
				// jump to specific index and continue search for variable
				i = i + indexEnter
				continue
			} else {
				// found a variable get variable name
				dollar := []byte(`$`)
				space := []byte(`=`)
				idxDollar := bytes.Index(data[i:], dollar)
				i = i + idxDollar
				idxSpace := bytes.Index(data[i:], space)
				varNameByte := data[i+1 : i+idxSpace]
				varName := removeSpace(string(varNameByte))

				//check if it is an array
				idxArray := bytes.Index(data[i:], []byte(`array`))
				if idxArray > -1 {
					i = i + idxArray
					// looop to get all the array
					varEndIndex := bytes.Index(data[i:], []byte(`;`))
					varEndIndex = i + varEndIndex
					index, values := getValues(i, varEndIndex, data)

					if len(values) > 0 {
						// found values insert to our map
						result[varName] = values
						i = index
					} else {
						// we did'nt found any values from the variable, skip it
						i = varEndIndex
					}

				}
			}

		} else {
			break
		}

	}
	return result
}

// Getting all values number from a variable.
// the number could be any data type like eg: int,float
// call this function if we got null values from getValuesString()
// Note i is current index in integer
func getValuesNumber(i, idxSemiColon int, data []byte) (int, []string) {
	var valuesNumber []string
	comma := []byte(`,`)
	oP := []byte(`(`) // oP -> open Parenthesis
	cP := []byte(`)`) // cP ->close Parenthesis
	var cPindex int
	var oPindex int
	var commaIndex int

	// finding the open parenthesis `(` character and assign tu current index = i
	oPindex = findChar(data, oP, i, idxSemiColon)

	// check oPindex
	if oPindex > -1 {
		i = oPindex + 1
	}

	if oPindex >= idxSemiColon || oPindex == -1 {
		return i, valuesNumber
	}

	for {
		commaIndex = findChar(data, comma, i, idxSemiColon)
		if commaIndex >= idxSemiColon {
			break
		}
		if commaIndex > -1 {
			// found separator get the value
			// set and check separatorIndex
			addValueStore(&valuesNumber, data, i, commaIndex)
			i = commaIndex + 1

			if i >= idxSemiColon {
				break
			}
		} else {
			// find close Parenthesis
			cPindex = findChar(data, cP, i, idxSemiColon)
			if cPindex >= idxSemiColon || cPindex == -1 {
				break
			}

			addValueStore(&valuesNumber, data, i, cPindex)
			i = cPindex + 1

		}

	}

	return i, valuesNumber
}

// Adding value to store from given data within the currentIndex and lastIndex
// remove the whitespaces if exist
// and return the currentIndex which modified from this function
func addValueStore(store *[]string, data []byte, currentIndex int, lastIndex int) {
	if currentIndex < lastIndex && lastIndex <= len(data) {
		value := data[currentIndex:lastIndex]
		if string(value) != "" {
			valueString := removeSpace(string(value))
			*store = append(*store, valueString)
		}
	}
}

// find the char index inside data from the given of currentIndex
// This currentIndex is to avoid getting previous charIndex
func findChar(data []byte, char []byte, currentIndex, idxSemiColon int) int {
	charIndex := -1
	if currentIndex < idxSemiColon && currentIndex < len(data) && idxSemiColon < len(data) {
		charIndex = bytes.Index(data[currentIndex:idxSemiColon], char)
		if charIndex > -1 {
			//found char  inside data
			charIndex = currentIndex + charIndex
		}
	}

	return charIndex
}

// Getting the values from data with index i to idxSemiColon
// In php you can use single "'" and double "'".
// This function will get the values concurrently for the string values
// if string values not exist then it will try to get the numbers
func getValues(i, idxSemiColon int, data []byte) (int, []string) {
	// type and variable to hold the result
	type storage struct {
		Index  int
		Values []string
	}
{'fraction_non_alphanumeric': 0.12258064516129032, 'fraction_numerical': 0.003089504770558837, 'mean_word_length': 3.269200930954228, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5337878', 'n_tokens_mistral': 4022, 'n_tokens_neox': 3523, 'n_words': 1230}