<commit_before>package de.domisum.lib.auxilium.contracts.storage;

import de.domisum.lib.auxilium.contracts.serialization.ToStringSerializer;
import de.domisum.lib.auxilium.util.FileUtil;
import de.domisum.lib.auxilium.util.FileUtil.FileType;
import de.domisum.lib.auxilium.util.java.annotations.API;
import lombok.RequiredArgsConstructor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Objects;
import java.util.Optional;

@API
@RequiredArgsConstructor
public class SerializedKeyableInDirectoryStorage<KeyT, T extends Keyable<KeyT>> implements Storage<KeyT, T>
{

	private final Logger logger = LoggerFactory.getLogger(getClass());


	// SETTINGS
	private final File directory;
	private final String fileExtension;
	private final ToStringSerializer<T> serializer;


	// SOURCE
	@Override public Collection<T> fetchAll()
	{
		Collection<File> files = FileUtil.listFilesRecursively(directory, FileType.FILE);

		Collection<File> validFiles = new ArrayList<>();
		for(File file : files)
			if(isFileExtensionValid(file))
				validFiles.add(file);

		return loadFromFiles(validFiles);
	}

	private boolean isFileExtensionValid(File file)
	{
		String extension = FileUtil.getCompositeExtension(file);
		boolean isValid = Objects.equals(fileExtension, extension);

		if(!isValid)
			logger.warn("Storage directory contains file with invalid extension ({}), skipping: {}", extension, file.getName());

		return isValid;
	}

	@API protected Collection<T> loadFromFiles(Collection<File> files)
	{
		Collection<T> items = new ArrayList<>();

		for(File file : files)
			if(isFileExtensionValid(file))
				items.add(readFromFile(file));

		return items;
	}

<commit_msg>Removed double check<commit_after>package de.domisum.lib.auxilium.contracts.storage;

import de.domisum.lib.auxilium.contracts.serialization.ToStringSerializer;
import de.domisum.lib.auxilium.util.FileUtil;
import de.domisum.lib.auxilium.util.FileUtil.FileType;
import de.domisum.lib.auxilium.util.java.annotations.API;
import lombok.RequiredArgsConstructor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Objects;
import java.util.Optional;

@API
@RequiredArgsConstructor
public class SerializedKeyableInDirectoryStorage<KeyT, T extends Keyable<KeyT>> implements Storage<KeyT, T>
{

	private final Logger logger = LoggerFactory.getLogger(getClass());


	// SETTINGS
	private final File directory;
	private final String fileExtension;
	private final ToStringSerializer<T> serializer;


	// SOURCE
	@Override public Collection<T> fetchAll()
	{
		Collection<File> files = FileUtil.listFilesRecursively(directory, FileType.FILE);

		Collection<File> validFiles = new ArrayList<>();
		for(File file : files)
			if(isFileExtensionValid(file))
				validFiles.add(file);

		return readFromFiles(validFiles);
	}

	private boolean isFileExtensionValid(File file)
	{
		String extension = FileUtil.getCompositeExtension(file);
		boolean isValid = Objects.equals(fileExtension, extension);

		if(!isValid)
			logger.warn("Storage directory contains file with invalid extension ({}), skipping: {}", extension, file.getName());

		return isValid;
	}

	@API protected Collection<T> readFromFiles(Collection<File> files)
	{
		Collection<T> items = new ArrayList<>();

		for(File file : files)
			items.add(readFromFile(file));

		return items;
	}


{'fraction_non_alphanumeric': 0.1095890410958904, 'fraction_numerical': 0.001141552511415525, 'mean_word_length': 5.71455938697318, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4560621', 'n_tokens_mistral': 1136, 'n_tokens_neox': 1065, 'n_words': 231}