<commit_before>/*
 * Copyright 2018, TeamDev. All rights reserved.
 *
 * Redistribution and use in source and/or binary forms, with or without
 * modification, must retain the above copyright notice and the following
 * disclaimer.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
package io.spine.code.proto;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.Streams;
import com.google.protobuf.DescriptorProtos.FileDescriptorProto;
import com.google.protobuf.DescriptorProtos.FileDescriptorSet;
import com.google.protobuf.ExtensionRegistry;
import io.spine.io.ResourceFiles;
import io.spine.option.OptionsProto;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.Iterator;
import java.util.List;
import java.util.function.Predicate;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static io.spine.util.Exceptions.newIllegalStateException;

/**
 * A utility class which allows to obtain Protobuf file descriptors.
 *
 * @author <NAME>
 * @author <NAME>
 */
public class FileDescriptors {

    /**
     * The name of the file generated by the Spine model compiler, which contains
     * a map from a proto type URL to corresponding Java class name.
     *
     *
     * // TODO:2018-06-05:dmytro.dashenkov: Update the doc.
     * <p>The file is generated during the build process of an application.
     */
    public static final String KNOWN_TYPES = "known_types.desc";

    private static final ExtensionRegistry EXTENSIONS = registerExtensions();

    /** Prevents instantiation of this utility class. */
    private FileDescriptors() {
    }

    /**
     * Returns descriptors of all {@code .proto} files described in the descriptor set file.
     *
     * @param descriptorSetFile the path to the file generated by the {@code protobuf-gradle-plugin}
     *                          which contains descriptors of the project {@code .proto} files
     * @return a list of descriptors
     */
    public static List<FileDescriptorProto> parse(String descriptorSetFile) {
        return parseAndFilter(descriptorSetFile, descriptor -> true);
    }

    /**
     * Obtains the list of files from the passed descriptor set file, skipping files provided
     * by Google Protobuf.
     */
    public static List<FileDescriptorProto> parseSkipStandard(String descriptorSetFile) {
        return parseAndFilter(descriptorSetFile, isNotGoogleProto());
    }

    /**
     * Returns descriptors of `.proto` files described in the descriptor set file
     * which match the filter predicate.
     *
     * @param descriptorSetFile the path to the file generated by {@code protobuf-gradle-plugin}
     *                          which contains descriptors of the project {@code .proto} files
     * @param filter            a filter predicate to apply to the files
     * @return a list of descriptors
     */
    private static List<FileDescriptorProto> parseAndFilter(String descriptorSetFile,
                                                            Predicate<FileDescriptorProto> filter) {
        final File descriptorsFile = new File(descriptorSetFile);
        checkArgument(descriptorsFile.exists(), "File %s does not exist", descriptorSetFile);

        final Logger log = log();
        if (log.isDebugEnabled()) {
            log.debug("Looking up for the proto files matching predicate {} under {}",
                      filter,
                      descriptorSetFile);
        }

        final ImmutableList.Builder<FileDescriptorProto> files = ImmutableList.builder();
        try (final FileInputStream fis = new FileInputStream(descriptorsFile)) {
            final FileDescriptorSet fileSet = FileDescriptorSet.parseFrom(fis);
            for (FileDescriptorProto file : fileSet.getFileList()) {
                if (filter.test(file)) {
                    files.add(file);
                }
            }
        } catch (IOException e) {
            throw newIllegalStateException(
                    e, "Cannot get proto file descriptors. Path: %s", descriptorSetFile
            );
        }

        final ImmutableList<FileDescriptorProto> result = files.build();
        log.debug("Found {} files: {}", result.size(), files);
        return result;
    }

    /**
     * Loads main file descriptor set from resources.
     */
    public static Iterator<FileDescriptorProto> load() {
        Iterator<URL> resources = ResourceFiles.loadAll(KNOWN_TYPES);
        Iterator<FileDescriptorProto> files = Streams.stream(resources)
                                                     .map(FileDescriptors::loadFrom)
                                                     .flatMap(set -> set.getFileList().stream())
                                                     .distinct()
                                                     .iterator();
        return files;
    }

    /**
     * Obtains the predicate that filters out file descriptors for types with {@code "google"}
     * in the package name.
     */
    public static Predicate<FileDescriptorProto> isNotGoogleProto() {
        return IsNotGoogleProto.PREDICATE;
    }

    private static FileDescriptorSet loadFrom(URL file) {
        checkNotNull(file);
        try {
            final InputStream stream = file.openStream();
            final FileDescriptorSet parsed = FileDescriptorSet.parseFrom(stream, EXTENSIONS);
            return parsed;
        } catch (IOException e) {
            throw newIllegalStateException(
                    e,
                    "Unable to load descriptor file set from %s.",
                    file
            );
        }
    }

    private static ExtensionRegistry registerExtensions() {
        ExtensionRegistry registry = ExtensionRegistry.newInstance();
        OptionsProto.registerAllExtensions(registry);
        return registry;
    }

    /**
     * Verifies if a package of a file does not contain {@code "google"} in its path.
     */
    private enum IsNotGoogleProto implements Predicate<FileDescriptorProto> {
        PREDICATE;

        private static final String GOOGLE_PACKAGE = "google";

        @Override
        public boolean test(FileDescriptorProto file) {
            checkNotNull(file);
            final boolean result = !file.getPackage()
                                        .contains(GOOGLE_PACKAGE);
            return result;
        }

        @Override
        public String toString() {
            return getClass().getSimpleName();
        }
    }

    private static Logger log() {
        return LogSingleton.INSTANCE.value;
    }

    private enum LogSingleton {
        INSTANCE;
        @SuppressWarnings("NonSerializableFieldInSerializableClass")
        private final Logger value = LoggerFactory.getLogger(FileDescriptors.class);
    }
}
<commit_msg>Close stream after read.<commit_after>/*
 * Copyright 2018, TeamDev. All rights reserved.
 *
 * Redistribution and use in source and/or binary forms, with or without
 * modification, must retain the above copyright notice and the following
 * disclaimer.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
package io.spine.code.proto;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.Streams;
import com.google.protobuf.DescriptorProtos.FileDescriptorProto;
import com.google.protobuf.DescriptorProtos.FileDescriptorSet;
import com.google.protobuf.ExtensionRegistry;
import io.spine.io.ResourceFiles;
import io.spine.option.OptionsProto;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.Iterator;
import java.util.List;
import java.util.function.Predicate;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static io.spine.util.Exceptions.newIllegalStateException;

/**
 * A utility class which allows to obtain Protobuf file descriptors.
 *
 * @author <NAME>
 * @author <NAME>
 */
public class FileDescriptors {

    /**
     * The name of the file generated by the Spine model compiler, which contains
     * a map from a proto type URL to corresponding Java class name.
     *
     *
     * // TODO:2018-06-05:dmytro.dashenkov: Update the doc.
     * <p>The file is generated during the build process of an application.
     */
    public static final String KNOWN_TYPES = "known_types.desc";

    private static final ExtensionRegistry EXTENSIONS = registerExtensions();

    /** Prevents instantiation of this utility class. */
    private FileDescriptors() {
    }

    /**
     * Returns descriptors of all {@code .proto} files described in the descriptor set file.
     *
     * @param descriptorSetFile the path to the file generated by the {@code protobuf-gradle-plugin}
     *                          which contains descriptors of the project {@code .proto} files
     * @return a list of descriptors
     */
    public static List<FileDescriptorProto> parse(String descriptorSetFile) {
        return parseAndFilter(descriptorSetFile, descriptor -> true);
    }

    /**
     * Obtains the list of files from the passed descriptor set file, skipping files provided
     * by Google Protobuf.
     */
    public static List<FileDescriptorProto> parseSkipStandard(String descriptorSetFile) {
        return parseAndFilter(descriptorSetFile, isNotGoogleProto());
    }

    /**
     * Returns descriptors of `.proto` files described in the descriptor set file
     * which match the filter predicate.
     *
     * @param descriptorSetFile the path to the file generated by {@code protobuf-gradle-plugin}
     *                          which contains descriptors of the project {@code .proto} files
     * @param filter            a filter predicate to apply to the files
     * @return a list of descriptors
     */
    private static List<FileDescriptorProto> parseAndFilter(String descriptorSetFile,
                                                            Predicate<FileDescriptorProto> filter) {
        final File descriptorsFile = new File(descriptorSetFile);
        checkArgument(descriptorsFile.exists(), "File %s does not exist", descriptorSetFile);

        final Logger log = log();
        if (log.isDebugEnabled()) {
            log.debug("Looking up for the proto files matching predicate {} under {}",
                      filter,
                      descriptorSetFile);
        }

        final ImmutableList.Builder<FileDescriptorProto> files = ImmutableList.builder();
        try (final FileInputStream fis = new FileInputStream(descriptorsFile)) {
            final FileDescriptorSet fileSet = FileDescriptorSet.parseFrom(fis);
            for (FileDescriptorProto file : fileSet.getFileList()) {
                if (filter.test(file)) {
                    files.add(file);
                }
            }
        } catch (IOException e) {
            throw newIllegalStateException(
                    e, "Cannot get proto file descriptors. Path: %s", descriptorSetFile
            );
        }

        final ImmutableList<FileDescriptorProto> result = files.build();
        log.debug("Found {} files: {}", result.size(), files);
        return result;
    }

    /**
     * Loads main file descriptor set from resources.
     */
    public static Iterator<FileDescriptorProto> load() {
        Iterator<URL> resources = ResourceFiles.loadAll(KNOWN_TYPES);
        Iterator<FileDescriptorProto> files = Streams.stream(resources)
                                                     .map(FileDescriptors::loadFrom)
                                                     .flatMap(set -> set.getFileList().stream())
                                                     .distinct()
                                                     .iterator();
        return files;
    }

    /**
     * Obtains the predicate that filters out file descriptors for types with {@code "google"}
     * in the package name.
     */
    public static Predicate<FileDescriptorProto> isNotGoogleProto() {
        return IsNotGoogleProto.PREDICATE;
    }

    private static FileDescriptorSet loadFrom(URL file) {
        checkNotNull(file);
        try (InputStream stream = file.openStream()) {
            final FileDescriptorSet parsed = FileDescriptorSet.parseFrom(stream, EXTENSIONS);
            return parsed;
        } catch (IOException e) {
            throw newIllegalStateException(
                    e,
                    "Unable to load file descriptor set from %s.",
                    file
            );
        }
    }

    private static ExtensionRegistry registerExtensions() {
        ExtensionRegistry registry = ExtensionRegistry.newInstance();
        OptionsProto.registerAllExtensions(registry);
        return registry;
    }

    /**
     * Verifies if a package of a file does not contain {@code "google"} in its path.
     */
    private enum IsNotGoogleProto implements Predicate<FileDescriptorProto> {
        PREDICATE;

        private static final String GOOGLE_PACKAGE = "google";

        @Override
        public boolean test(FileDescriptorProto file) {
            checkNotNull(file);
            final boolean result = !file.getPackage()
                                        .contains(GOOGLE_PACKAGE);
            return result;
        }

        @Override
        public String toString() {
            return getClass().getSimpleName();
        }
    }

    private static Logger log() {
        return LogSingleton.INSTANCE.value;
    }

    private enum LogSingleton {
        INSTANCE;
        @SuppressWarnings("NonSerializableFieldInSerializableClass")
        private final Logger value = LoggerFactory.getLogger(FileDescriptors.class);
    }
}

{'fraction_non_alphanumeric': 0.07811491284699806, 'fraction_numerical': 0.0018076178179470626, 'mean_word_length': 2.4332890070921986, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '529491', 'n_tokens_mistral': 3873, 'n_tokens_neox': 3672, 'n_words': 1252}