<commit_before>package com.metawiring.gen.util;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.nio.CharBuffer;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class FileReaders {
    private static final Logger logger = LoggerFactory.getLogger(FileReaders.class);

    public static List<String> loadToStringList(String resource) {
        CharBuffer charBuffer = loadFileToCharBuffer(resource);
        String[] split = charBuffer.toString().split("\\r*\\n");
        List<String> strings = Arrays.asList(split);
        return Collections.unmodifiableList(strings);
    }

    public static CharBuffer loadFileToCharBuffer(String filename, String... searchPaths) {

        InputStream stream = mapInputResource(filename, searchPaths);

        CharBuffer linesImage;
        try {
            InputStreamReader isr = new InputStreamReader(stream);
            linesImage = CharBuffer.allocate(1024 * 1024);
            while (isr.read(linesImage)>0) {
            }
            isr.close();
        } catch (IOException e) {
            logger.error(e.getMessage());
            throw new RuntimeException(e);
        }
        linesImage.flip();
        return linesImage.asReadOnlyBuffer();
    }

    private static InputStream mapInputResource(String name, String... searchPaths) {
        InputStream stream = null;

        for (String searchPath : searchPaths) {
            File onFileSystem = new File( searchPath + File.separator + name);

            if (onFileSystem.exists()) {
                try {
                    stream = new FileInputStream(onFileSystem);
                    logger.debug("Found resource on filesystem:" + onFileSystem.getPath());
                    return stream;
                } catch (FileNotFoundException e) {
                    throw new RuntimeException("Unable to find file " + onFileSystem.getPath() + " after verifying that it exists.");
                }
            }
        }

        ClassLoader cl = Thread.currentThread().getContextClassLoader();

        for (String searchPath : searchPaths) {
            String inClassPath = searchPath + File.separator + name;
            stream = cl.getResourceAsStream(inClassPath);
            if (stream!=null) {
                logger.debug("Found resource in class path:" + inClassPath);
                return stream;
            }
        }
<commit_msg>updated classloader in FileReaders<commit_after>package com.metawiring.gen.util;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.nio.CharBuffer;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class FileReaders {
    private static final Logger logger = LoggerFactory.getLogger(FileReaders.class);

    public static List<String> loadToStringList(String resource) {
        CharBuffer charBuffer = loadFileToCharBuffer(resource);
        String[] split = charBuffer.toString().split("\\r*\\n");
        List<String> strings = Arrays.asList(split);
        return Collections.unmodifiableList(strings);
    }

    public static CharBuffer loadFileToCharBuffer(String filename, String... searchPaths) {

        InputStream stream = mapInputResource(filename, searchPaths);

        CharBuffer linesImage;
        try {
            InputStreamReader isr = new InputStreamReader(stream);
            linesImage = CharBuffer.allocate(1024 * 1024);
            while (isr.read(linesImage)>0) {
            }
            isr.close();
        } catch (IOException e) {
            logger.error(e.getMessage());
            throw new RuntimeException(e);
        }
        linesImage.flip();
        return linesImage.asReadOnlyBuffer();
    }

    private static InputStream mapInputResource(String name, String... searchPaths) {
        InputStream stream = null;

        for (String searchPath : searchPaths) {
            File onFileSystem = new File( searchPath + File.separator + name);

            if (onFileSystem.exists()) {
                try {
                    stream = new FileInputStream(onFileSystem);
                    logger.debug("Found resource on filesystem:" + onFileSystem.getPath());
                    return stream;
                } catch (FileNotFoundException e) {
                    throw new RuntimeException("Unable to find file " + onFileSystem.getPath() + " after verifying that it exists.");
                }
            }
        }

        ClassLoader cl = FileReaders.class.getClassLoader();

        for (String searchPath : searchPaths) {
            String inClassPath = searchPath + File.separator + name;
            stream = cl.getResourceAsStream(inClassPath);
            if (stream!=null) {
                logger.debug("Found resource in class path:" + inClassPath);
                return stream;
            }
        }

{'fraction_non_alphanumeric': 0.09568595379268044, 'fraction_numerical': 0.004498057656920875, 'mean_word_length': 2.2569906790945407, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '681011', 'n_tokens_mistral': 1263, 'n_tokens_neox': 1202, 'n_words': 299}