<commit_before>/*
 * Copyright (c) 2002-2016 "Neo Technology,"
 * Network Engine for Objects in Lund AB [http://neotechnology.com]
 *
 * This product is licensed to you under the Apache License, Version 2.0 (the "License").
 * You may not use this product except in compliance with the License.
 *
 * This product may include a number of subcomponents with
 * separate copyright notices and license terms. Your use of the source
 * code for these subcomponents is subject to the terms and
 *  conditions of the subcomponent's license, as noted in the LICENSE file.
 */

package org.neo4j.ogm.scanner;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Enumeration;
import java.util.List;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;

import org.neo4j.ogm.ClassUtils;
import org.neo4j.ogm.metadata.ClassFileProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * @author <NAME>
 * @author <NAME>
 */
public class ClassPathScanner {

    private List<String> classPaths;
    private ClassFileProcessor processor;

    private static final Logger LOGGER = LoggerFactory.getLogger(ClassPathScanner.class);


    private void scanFile(File file, String relativePath) throws IOException {
        if (relativePath.endsWith(".class")) {
            try (InputStream inputStream = new FileInputStream(file)) {
                processor.process(inputStream);
            }
        }
    }

    private void scanFolder(File folder, int prefixSize) throws IOException {

        String absolutePath = folder.getPath();
        String relativePath = prefixSize > absolutePath.length() ? "" : absolutePath.substring(prefixSize);

        File[] subFiles = folder.listFiles();
        if (subFiles != null) {
            for (final File subFile : subFiles) {
                if (subFile.isDirectory()) {
                    scanFolder(subFile, prefixSize);
                } else if (subFile.isFile()) {
                    String leafSuffix = "/" + subFile.getName();
                    scanFile(subFile, relativePath + leafSuffix);
                }
            }
        }
    }


    private void scanZipFile(final ZipFile zipFile) throws IOException {
        LOGGER.debug("Scanning zipFile {}", zipFile.getName());
        for (Enumeration<? extends ZipEntry> entries = zipFile.entries(); entries.hasMoreElements();) {
            final ZipEntry entry = entries.nextElement();
            if (!entry.isDirectory()) {
                try (InputStream inputStream = zipFile.getInputStream(entry) ) {
                    if (entry.getName().endsWith(".class")) {
                        scanClassFileEntry(inputStream, entry);
                    } else if (entry.getName().endsWith(".jar") || entry.getName().endsWith(".zip") || entry.getName().endsWith(".war")) {
                        scanZippedEntry(inputStream, entry);
                    }
                }
            }
        }
    }

    private void scanClassFileEntry(InputStream inputStream, ZipEntry entry) throws IOException {

        String name = entry.getName();

        LOGGER.debug("Scanning class entry: {}", name);
        int i = name.lastIndexOf("/");
        String path = (i == -1) ? "" : name.substring(0, i);

        for (String pathToScan : classPaths) {
            if (path.equals(pathToScan) || path.contains(pathToScan)) {
                LOGGER.debug("{} admits {} for entry: {}", new Object[] {pathToScan, path ,name});
                processor.process(inputStream);
                break;
            }
        }
    }

    private void scanZippedEntry(InputStream inputStream, ZipEntry entry) throws IOException {

        String name = entry.getName();

        LOGGER.debug("Scanning zipped entry: {}", name);
        ZipInputStream zipInputStream = new ZipInputStream(inputStream);

        ZipEntry zipEntry = zipInputStream.getNextEntry();

        while (zipEntry != null) {
            if (!zipEntry.isDirectory()) {
                if (zipEntry.getName().endsWith(".class")) {
                    scanClassFileEntry(zipInputStream, zipEntry);
                } else if (zipEntry.getName().endsWith(".jar") || zipEntry.getName().endsWith(".zip") || zipEntry.getName().endsWith(".war")) {
                    scanZippedEntry(zipInputStream, zipEntry);
                }
            }
            zipEntry = zipInputStream.getNextEntry();
        }
    }

    public void scan(List<String> classPaths, ClassFileProcessor processor) {

        this.classPaths = classPaths;
        this.processor = processor;

        Set<File> classPathElements = getUniqueClasspathElements(classPaths);

        LOGGER.debug("Classpath elements:");
        for (File classPathElement : classPathElements) {
            LOGGER.debug(classPathElement.getPath());
        }

        try {
            for (File classPathElement : classPathElements) {
                String path = classPathElement.getPath();
                if (classPathElement.isDirectory()) {
                    scanFolder(classPathElement, path.length() + 1);
                } else if (classPathElement.isFile()) {
                    String pathLower = path.toLowerCase();
                    if (pathLower.endsWith(".jar") || pathLower.endsWith(".zip") || pathLower.endsWith(".war")) {
                        scanZipFile(new ZipFile(classPathElement));
                    } else {
                        scanFile(classPathElement, classPathElement.getName());
                    }
                }
            }
            processor.finish();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    protected Set<File> getUniqueClasspathElements(List<String> classPaths) {
        return ClassUtils.getUniqueClasspathElements(classPaths);
    }

}<commit_msg>Revert DATAGRAPH-849 packages in an executable war were not scanned<commit_after>/*
 * Copyright (c) 2002-2016 "Neo Technology,"
 * Network Engine for Objects in Lund AB [http://neotechnology.com]
 *
 * This product is licensed to you under the Apache License, Version 2.0 (the "License").
 * You may not use this product except in compliance with the License.
 *
 * This product may include a number of subcomponents with
 * separate copyright notices and license terms. Your use of the source
 * code for these subcomponents is subject to the terms and
 *  conditions of the subcomponent's license, as noted in the LICENSE file.
 */

package org.neo4j.ogm.scanner;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Enumeration;
import java.util.List;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;

import org.neo4j.ogm.ClassUtils;
import org.neo4j.ogm.metadata.ClassFileProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * @author <NAME>
 * @author <NAME>
 */
public class ClassPathScanner {

    private List<String> classPaths;
    private ClassFileProcessor processor;

    private static final Logger LOGGER = LoggerFactory.getLogger(ClassPathScanner.class);


    private void scanFile(File file, String relativePath) throws IOException {
        if (relativePath.endsWith(".class")) {
            try (InputStream inputStream = new FileInputStream(file)) {
                processor.process(inputStream);
            }
        }
    }

    private void scanFolder(File folder, int prefixSize) throws IOException {

        String absolutePath = folder.getPath();
        String relativePath = prefixSize > absolutePath.length() ? "" : absolutePath.substring(prefixSize);

        File[] subFiles = folder.listFiles();
        if (subFiles != null) {
            for (final File subFile : subFiles) {
                if (subFile.isDirectory()) {
                    scanFolder(subFile, prefixSize);
                } else if (subFile.isFile()) {
                    String leafSuffix = "/" + subFile.getName();
                    scanFile(subFile, relativePath + leafSuffix);
                }
            }
        }
    }


    private void scanZipFile(final ZipFile zipFile) throws IOException {
        LOGGER.debug("Scanning zipFile {}", zipFile.getName());
        for (Enumeration<? extends ZipEntry> entries = zipFile.entries(); entries.hasMoreElements();) {
            final ZipEntry entry = entries.nextElement();
            if (!entry.isDirectory()) {
                try (InputStream inputStream = zipFile.getInputStream(entry) ) {
                    if (entry.getName().endsWith(".class")) {
                        scanClassFileEntry(inputStream, entry);
                    } else if (entry.getName().endsWith(".jar") || entry.getName().endsWith(".zip")) {
                        scanZippedEntry(inputStream, entry);
                    }
                }
            }
        }
    }

    private void scanClassFileEntry(InputStream inputStream, ZipEntry entry) throws IOException {

        String name = entry.getName();

        LOGGER.debug("Scanning class entry: {}", name);
        int i = name.lastIndexOf("/");
        String path = (i == -1) ? "" : name.substring(0, i);

        for (String pathToScan : classPaths) {
            if (path.equals(pathToScan) || path.startsWith(pathToScan.concat("/"))) {
                LOGGER.debug("{} admits {} for entry: {}", new Object[] {pathToScan, path ,name});
                processor.process(inputStream);
                break;
            }
        }
    }

    private void scanZippedEntry(InputStream inputStream, ZipEntry entry) throws IOException {

        String name = entry.getName();

        LOGGER.debug("Scanning zipped entry: {}", name);
        ZipInputStream zipInputStream = new ZipInputStream(inputStream);

        ZipEntry zipEntry = zipInputStream.getNextEntry();

        while (zipEntry != null) {
            if (!zipEntry.isDirectory()) {
                if (zipEntry.getName().endsWith(".class")) {
                    scanClassFileEntry(zipInputStream, zipEntry);
                } else if (zipEntry.getName().endsWith(".jar") || zipEntry.getName().endsWith(".zip")) {
                    scanZippedEntry(zipInputStream, zipEntry);
                }
            }
            zipEntry = zipInputStream.getNextEntry();
        }
    }

    public void scan(List<String> classPaths, ClassFileProcessor processor) {

        this.classPaths = classPaths;
        this.processor = processor;

        Set<File> classPathElements = getUniqueClasspathElements(classPaths);

        LOGGER.debug("Classpath elements:");
        for (File classPathElement : classPathElements) {
            LOGGER.debug(classPathElement.getPath());
        }

        try {
            for (File classPathElement : classPathElements) {
                String path = classPathElement.getPath();
                if (classPathElement.isDirectory()) {
                    scanFolder(classPathElement, path.length() + 1);
                } else if (classPathElement.isFile()) {
                    String pathLower = path.toLowerCase();
                    if (pathLower.endsWith(".jar") || pathLower.endsWith(".zip")) {
                        scanZipFile(new ZipFile(classPathElement));
                    } else {
                        scanFile(classPathElement, classPathElement.getName());
                    }
                }
            }
            processor.finish();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    protected Set<File> getUniqueClasspathElements(List<String> classPaths) {
        return ClassUtils.getUniqueClasspathElements(classPaths);
    }

}
{'fraction_non_alphanumeric': 0.10894776940658596, 'fraction_numerical': 0.0033014475577753323, 'mean_word_length': 2.2626346313173156, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7658975', 'n_tokens_mistral': 3116, 'n_tokens_neox': 2962, 'n_words': 740}