<commit_before>package org.apache.logging.log4j.core.script;

import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URI;
import java.nio.charset.Charset;

import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.config.Node;
import org.apache.logging.log4j.core.config.plugins.Plugin;
import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
import org.apache.logging.log4j.core.config.plugins.PluginFactory;
import org.apache.logging.log4j.core.util.FileUtils;
import org.apache.logging.log4j.core.util.IOUtils;
import org.apache.logging.log4j.core.util.NetUtils;
import org.apache.logging.log4j.status.StatusLogger;

/**
 * Container for the language and body of a script file.
 */
@Plugin(name = "ScriptFile", category = Node.CATEGORY, printObject = true)
public class ScriptFile extends AbstractScript {

    private static final Logger logger = StatusLogger.getLogger();

    public ScriptFile(String name, String language, String scriptText) {
        super(name, language, scriptText);
    }

    @PluginFactory
    public static ScriptFile createScript(
            // @formatter:off
            @PluginAttribute("name") final String name,
            @PluginAttribute("language") String language, 
            @PluginAttribute("path") final String filePathOrUri,
            @PluginAttribute("charset") final Charset charset) {
            // @formatter:on
        if (language == null) {
            logger.info("No script language supplied, defaulting to {}", DEFAULT_LANGUAGE);
            language = DEFAULT_LANGUAGE;
        }
        if (filePathOrUri == null) {
            logger.error("No script path provided for ScriptFile {}", name);
            return null;
        }
        final Charset actualCharset = charset == null ? Charset.defaultCharset() : charset;
        final URI uri = NetUtils.toURI(filePathOrUri);
        final File file = FileUtils.fileFromUri(uri);
        String scriptText;
        try (final Reader reader = file != null ? new FileReader(file)
                : new InputStreamReader(uri.toURL().openStream(), actualCharset)) {
            scriptText = IOUtils.toString(reader);
        } catch (IOException e) {
            logger.error("{}: name={}, language={}, path={}, actualCharset={}", e.getClass().getSimpleName(), name,
                    language, filePathOrUri, actualCharset);
            return null;
        }
        return new ScriptFile(name, language, scriptText);

    }
}
<commit_msg>[LOG4J2-1136] Add support for JSR 223 scripts in filters and the PatternSelector: Use charset for files too.<commit_after>package org.apache.logging.log4j.core.script;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URI;
import java.nio.charset.Charset;

import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.config.Node;
import org.apache.logging.log4j.core.config.plugins.Plugin;
import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
import org.apache.logging.log4j.core.config.plugins.PluginFactory;
import org.apache.logging.log4j.core.util.FileUtils;
import org.apache.logging.log4j.core.util.IOUtils;
import org.apache.logging.log4j.core.util.NetUtils;
import org.apache.logging.log4j.status.StatusLogger;

/**
 * Container for the language and body of a script file.
 */
@Plugin(name = "ScriptFile", category = Node.CATEGORY, printObject = true)
public class ScriptFile extends AbstractScript {

    private static final Logger logger = StatusLogger.getLogger();

    public ScriptFile(String name, String language, String scriptText) {
        super(name, language, scriptText);
    }

    @PluginFactory
    public static ScriptFile createScript(
            // @formatter:off
            @PluginAttribute("name") final String name,
            @PluginAttribute("language") String language, 
            @PluginAttribute("path") final String filePathOrUri,
            @PluginAttribute("charset") final Charset charset) {
            // @formatter:on
        if (language == null) {
            logger.info("No script language supplied, defaulting to {}", DEFAULT_LANGUAGE);
            language = DEFAULT_LANGUAGE;
        }
        if (filePathOrUri == null) {
            logger.error("No script path provided for ScriptFile {}", name);
            return null;
        }
        final Charset actualCharset = charset == null ? Charset.defaultCharset() : charset;
        final URI uri = NetUtils.toURI(filePathOrUri);
        final File file = FileUtils.fileFromUri(uri);
        String scriptText;
        try (final Reader reader = new InputStreamReader(
                file != null ? new FileInputStream(file) : uri.toURL().openStream(), actualCharset)) {
            scriptText = IOUtils.toString(reader);
        } catch (IOException e) {
            logger.error("{}: name={}, language={}, path={}, actualCharset={}", e.getClass().getSimpleName(), name,
                    language, filePathOrUri, actualCharset);
            return null;
        }
        return new ScriptFile(name, language, scriptText);

    }
}

{'fraction_non_alphanumeric': 0.1112167300380228, 'fraction_numerical': 0.005513307984790874, 'mean_word_length': 3.11015625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '261546', 'n_tokens_mistral': 1470, 'n_tokens_neox': 1452, 'n_words': 351}