<commit_before>package comparer

import (
	"bufio"
	"os"
	"strings"
)

// readFile receives a complete path of a file and returns It's tags & an error
func readFile(file, path string) ([]string, error) {
	inFile, err := os.Open(path + file)
	if err != nil {
		return nil, err
	}
	defer inFile.Close()
	tags := []string{}
	scanner := bufio.NewScanner(inFile)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		indexStart := strings.Index(line, "<")
		indexEnd := strings.Index(line, ">")
		if (indexStart != -1) && (indexEnd != -1) {
			tags = append(tags, line[indexStart:indexEnd+1])
		}
	}
	return tags, nil
}

func readFiles(original, translation string) error {
	_, filesOri, err := directoriesAndFiles(original)
	if err != nil {
		return err
	}
	original = original + "/"
	translation = translation + "/"
	for _, file := range filesOri {
		tagsA, err := readFile(file, original)
		if err != nil {
			return err
		}
		tagsB, err := readFile(file, translation)
		if (err != nil) || (tagsB == nil) {
			continue
		}
		fileSplited := strings.Split(file, ".")
		fileName := fileSplited[0] + "_"
		missing := findMissing(tagsA, tagsB)
		err = createOutuputFile(translation, fileName, "MissingTags.txt", missing)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>reader: func to check last char of path<commit_after>package comparer

import (
	"bufio"
	"os"
	"strings"
)

// readFile receives a complete path of a file and returns It's tags & an error
func readFile(file, path string) ([]string, error) {
	inFile, err := os.Open(path + file)
	if err != nil {
		return nil, err
	}
	defer inFile.Close()
	tags := []string{}
	scanner := bufio.NewScanner(inFile)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		indexStart := strings.Index(line, "<")
		indexEnd := strings.Index(line, ">")
		if (indexStart != -1) && (indexEnd != -1) {
			tags = append(tags, line[indexStart:indexEnd+1])
		}
	}
	return tags, nil
}

func readFiles(original, translation string) error {
	_, filesOri, err := directoriesAndFiles(original)
	if err != nil {
		return err
	}
	original = lastChar(original)
	translation = lastChar(translation)
	for _, file := range filesOri {
		tagsA, err := readFile(file, original)
		if err != nil {
			return err
		}
		tagsB, err := readFile(file, translation)
		if (err != nil) || (tagsB == nil) {
			continue
		}
		fileSplited := strings.Split(file, ".")
		fileName := fileSplited[0] + "_"
		missing := findMissing(tagsA, tagsB)
		err = createOutuputFile(translation, fileName, "MissingTags.txt", missing)
		if err != nil {
			return err
		}
	}
	return nil
}

func lastChar(s string) string {
	if s[len(s)-1:] != "/" {
		s = s + "/"
	}
	return s
}

{'fraction_non_alphanumeric': 0.15173674588665448, 'fraction_numerical': 0.0032906764168190127, 'mean_word_length': 3.7010309278350517, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2084001', 'n_tokens_mistral': 1034, 'n_tokens_neox': 969, 'n_words': 259}