<commit_before>/**
 * redpen: a text inspection tool
 * Copyright (c) 2014-2015 Recruit Technologies Co., Ltd. and contributors
 * (see CONTRIBUTORS.md)
 * <p>
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * <p>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p>
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package cc.redpen.util;

import cc.redpen.RedPenException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.function.BiConsumer;
import java.util.function.Supplier;

/**
 * ResourceExtractor is called from FileLoader. To support a file format,
 * we create a class implementing ResourceExtractor.
 */
public final class ResourceExtractor<E> {
    private static final Logger LOG = LoggerFactory.getLogger(ResourceExtractor.class);

    private final Supplier<E> supplier;
    private final BiConsumer<E, String> loader;

    private ResourceExtractor(Supplier<E> supplier, BiConsumer<E, String> loader) {
        this.supplier = supplier;
        this.loader = loader;
    }

    /**
     * Resource Extractor loads key-value dictionary
     */
    public final static ResourceExtractor<Map<String, String>> KEY_VALUE_DICTIONARY =
            new ResourceExtractor<>(HashMap::new, (map, line) -> {
                String[] result = line.split("\t");
                if (result.length == 2) {
                    map.put(result[0], result[1]);
                } else {
                    LOG.error("Skip to load line... Invalid line: " + line);
                }
            });

    /**
     * Resource Extractor loads word list
     */
    public final static ResourceExtractor<Set<String>> WORD_LIST =
            new ResourceExtractor<>(HashSet::new, Set::add);

    /**
     * Resource Extractor loads word list while lowercasing lines
     */
    public final static ResourceExtractor<Set<String>> WORD_LIST_LOWERCASE =
            new ResourceExtractor<>(HashSet::new, (set, line) -> set.add(line.toLowerCase()));

    /**
     * Given a input stream, load the contents.
     *
     * @param inputStream input stream
     * @throws IOException when failed to create reader from the specified input stream
     */
    public E load(InputStream inputStream) throws IOException {
        E e = supplier.get();
        try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream,
                StandardCharsets.UTF_8))) {
            bufferedReader.lines().forEach(line -> loader.accept(e, line));
        }
        return e;
    }

    /**
     * Load a given input file combined with jar package.
     *
     * @param inputFile a file included in the jar file
     * @throws IOException when input stream is null
     */
    public E loadFromResource(String inputFile) throws IOException {
        try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream(inputFile)) {
            if (inputStream == null) {
                throw new IOException("Failed to load input " + inputFile);
            }
            return load(inputStream);
        }
    }

    private final Map<String,E> wordListCache = new HashMap<>();

    /**
     * returns word list loaded from resource
     * @param path resource path
     * @param dictionaryName name of the resource
     * @return word list
     * @throws RedPenException
     */
    public E loadCachedFromResource(String path, String dictionaryName) throws RedPenException {
        E strings = wordListCache.computeIfAbsent(path, e -> {
            try {
                return loadFromResource(path);
            } catch (IOException ioe) {
                LOG.error(ioe.getMessage());
                return null;
            }
        });
        if (strings == null) {<commit_msg>rename refactoring<commit_after>/**
 * redpen: a text inspection tool
 * Copyright (c) 2014-2015 Recruit Technologies Co., Ltd. and contributors
 * (see CONTRIBUTORS.md)
 * <p>
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * <p>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p>
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package cc.redpen.util;

import cc.redpen.RedPenException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.function.BiConsumer;
import java.util.function.Supplier;

/**
 * ResourceExtractor is called from FileLoader. To support a file format,
 * we create a class implementing ResourceExtractor.
 */
public final class ResourceExtractor<E> {
    private static final Logger LOG = LoggerFactory.getLogger(ResourceExtractor.class);

    private final Supplier<E> supplier;
    private final BiConsumer<E, String> loader;

    private ResourceExtractor(Supplier<E> supplier, BiConsumer<E, String> loader) {
        this.supplier = supplier;
        this.loader = loader;
    }

    /**
     * Resource Extractor loads key-value dictionary
     */
    public final static ResourceExtractor<Map<String, String>> KEY_VALUE_DICTIONARY =
            new ResourceExtractor<>(HashMap::new, (map, line) -> {
                String[] result = line.split("\t");
                if (result.length == 2) {
                    map.put(result[0], result[1]);
                } else {
                    LOG.error("Skip to load line... Invalid line: " + line);
                }
            });

    /**
     * Resource Extractor loads word list
     */
    public final static ResourceExtractor<Set<String>> WORD_LIST =
            new ResourceExtractor<>(HashSet::new, Set::add);

    /**
     * Resource Extractor loads word list while lowercasing lines
     */
    public final static ResourceExtractor<Set<String>> WORD_LIST_LOWERCASE =
            new ResourceExtractor<>(HashSet::new, (set, line) -> set.add(line.toLowerCase()));

    /**
     * Given a input stream, load the contents.
     *
     * @param inputStream input stream
     * @throws IOException when failed to create reader from the specified input stream
     */
    public E load(InputStream inputStream) throws IOException {
        E e = supplier.get();
        try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream,
                StandardCharsets.UTF_8))) {
            bufferedReader.lines().forEach(line -> loader.accept(e, line));
        }
        return e;
    }

    /**
     * Load a given input file combined with jar package.
     *
     * @param inputFile a file included in the jar file
     * @throws IOException when input stream is null
     */
    public E loadFromResource(String inputFile) throws IOException {
        try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream(inputFile)) {
            if (inputStream == null) {
                throw new IOException("Failed to load input " + inputFile);
            }
            return load(inputStream);
        }
    }

    private final Map<String,E> resourceCache = new HashMap<>();

    /**
     * returns word list loaded from resource
     * @param path resource path
     * @param dictionaryName name of the resource
     * @return word list
     * @throws RedPenException
     */
    public E loadCachedFromResource(String path, String dictionaryName) throws RedPenException {
        E strings = resourceCache.computeIfAbsent(path, e -> {
            try {
                return loadFromResource(path);
            } catch (IOException ioe) {
                LOG.error(ioe.getMessage());
                return null;
            }
        });
        if (strings == null) {
{'fraction_non_alphanumeric': 0.0955907881032288, 'fraction_numerical': 0.004166184469390117, 'mean_word_length': 2.6433389544688026, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6271443', 'n_tokens_mistral': 2274, 'n_tokens_neox': 2214, 'n_words': 743}