<commit_before>/**
 *  Copyright (C) 2004 Orbeon, Inc.
 *
 *  This program is free software; you can redistribute it and/or modify it under the terms of the
 *  GNU Lesser General Public License as published by the Free Software Foundation; either version
 *  2.1 of the License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 *  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *  See the GNU Lesser General Public License for more details.
 *
 *  The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
 */
package org.orbeon.oxf.processor.generator;

import com.drew.imaging.jpeg.JpegSegmentReader;
import com.drew.metadata.Directory;
import com.drew.metadata.Metadata;
import com.drew.metadata.MetadataException;
import com.drew.metadata.Tag;
import com.drew.metadata.exif.ExifReader;
import com.drew.metadata.iptc.IptcReader;
import org.dom4j.Document;
import org.dom4j.Node;
import org.orbeon.oxf.common.OXFException;
import org.orbeon.oxf.pipeline.api.PipelineContext;
import org.orbeon.oxf.processor.*;
import org.orbeon.oxf.util.ISODateUtils;
import org.orbeon.oxf.util.NumberUtils;
import org.orbeon.oxf.xml.ContentHandlerHelper;
import org.orbeon.oxf.xml.XPathUtils;
import org.orbeon.oxf.xml.dom4j.LocationData;
import org.orbeon.oxf.resources.URLFactory;
import org.orbeon.oxf.resources.ResourceManagerWrapper;
import org.xml.sax.ContentHandler;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URLConnection;
import java.net.URL;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;

/**
 * This processor scans a filesystem hierarchy and retrieves information about each file. The
 * implementation is based on a modified version of the ant directory scanner.
 *
 * Ideas for improvements:
 *
 * o componentize so that file identification and enhanced file information (such as images) can be
 *   plugged
 *
 * o you could imagine, for XML files, to extract content with XPath expressions
 */
public class DirectoryScannerProcessor extends ProcessorImpl {

    public static final String DIRECTORY_GENERATOR_NAMESPACE_URI = "http://www.orbeon.org/oxf/directory-generator";

    private static final String DIRECTORY_ELEMENT = "directory";
    private static final String FILE_ELEMENT = "file";

    private static final String EXIF_ELEMENT = "exif-info";
    private static final String IPTC_ELEMENT = "iptc-info";
    private static final String TAG_ELEMENT = "param";

    private static final boolean DEFAULT_CASE_SENSITIVE = true;
    private static final boolean DEFAULT_DEFAULT_EXCLUDES = false;

    private static final boolean DEFAULT_BASIC_INFO = false;
    private static final boolean DEFAULT_EXIF_INFO = false;
    private static final boolean DEFAULT_IPTC_INFO = false;


    public DirectoryScannerProcessor() {
        addInputInfo(new ProcessorInputOutputInfo(INPUT_CONFIG, DIRECTORY_GENERATOR_NAMESPACE_URI));
        addOutputInfo(new ProcessorInputOutputInfo(OUTPUT_DATA));
    }

    public ProcessorOutput createOutput(String name) {
        ProcessorOutput output = new ProcessorImpl.ProcessorOutputImpl(getClass(), name) {
            public void readImpl(PipelineContext context, ContentHandler contentHandler) {

                // Read config
                final Config config = (Config) readCacheInputAsObject(context, getInputByName(INPUT_CONFIG), new CacheableInputReader() {
                    public Object read(PipelineContext context, ProcessorInput input) {

                        final Document configNode = readInputAsDOM4J(context, input);
                        final Config config = new Config();

                        final String baseDirectoryURLString = XPathUtils.selectStringValueNormalize(configNode, "/config/base-directory").trim();

                        // Use location data if present so that relative URLs can be supported
                        final LocationData locationData = getLocationData();
                        final URL fullURL;
                        final String realPath;
                        try {
                            fullURL = (locationData != null && locationData.getSystemID() != null)
                                                            ? URLFactory.createURL(locationData.getSystemID(), baseDirectoryURLString)
                                                            : URLFactory.createURL(baseDirectoryURLString);

                            if (fullURL.getProtocol().equals("oxf")) {
                                // Get real path to resource path if possible
                                realPath = ResourceManagerWrapper.instance().getRealPath(fullURL.getFile());
                                if (realPath == null)
                                    throw new OXFException("Directory Scanner processor is unable to obtain the real path of the file using the oxf: protocol for the base-directory property: " + baseDirectoryURLString);
                            } else if (fullURL.getProtocol().equals("file")) {
                                realPath = fullURL.getFile();
                            } else {
                                throw new OXFException("Directory Scanner processor only supports the file: and oxf: protocols for the base-directory property: " + baseDirectoryURLString);
                            }

                        } catch (MalformedURLException e) {
                            throw new OXFException(e);
                        }
                        config.setBaseDirectory(realPath);

                        for (Iterator i = XPathUtils.selectIterator(configNode, "/config/include"); i.hasNext();) {
                            final Node node = (Node) i.next();
                            final String value = XPathUtils.selectStringValueNormalize(node, ".");
                            if (value != null)
                                config.addInclude(value);
                        }
                        for (Iterator i = XPathUtils.selectIterator(configNode, "/config/exclude"); i.hasNext();) {
                            final Node node = (Node) i.next();
                            final String value = XPathUtils.selectStringValueNormalize(node, ".");
                            if (value != null)
                                config.addExclude(value);
                        }
                        final boolean caseSensitive = ProcessorUtils.selectBooleanValue(configNode, "/config/case-sensitive", DEFAULT_CASE_SENSITIVE);
                        config.setCaseSensitive(caseSensitive);
                        final boolean defaultExcludes = ProcessorUtils.selectBooleanValue(configNode, "/config/default-excludes", DEFAULT_DEFAULT_EXCLUDES);
                        config.setDefaultExcludes(defaultExcludes);

                        final boolean basicInfo = ProcessorUtils.selectBooleanValue(configNode, "/config/image-metadata/basic-info", DEFAULT_BASIC_INFO);<commit_msg>Fix [ #304684 ] Exception when listing content of shared directory<commit_after>/**
 *  Copyright (C) 2004 Orbeon, Inc.
 *
 *  This program is free software; you can redistribute it and/or modify it under the terms of the
 *  GNU Lesser General Public License as published by the Free Software Foundation; either version
 *  2.1 of the License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 *  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *  See the GNU Lesser General Public License for more details.
 *
 *  The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
 */
package org.orbeon.oxf.processor.generator;

import com.drew.imaging.jpeg.JpegSegmentReader;
import com.drew.metadata.Directory;
import com.drew.metadata.Metadata;
import com.drew.metadata.MetadataException;
import com.drew.metadata.Tag;
import com.drew.metadata.exif.ExifReader;
import com.drew.metadata.iptc.IptcReader;
import org.dom4j.Document;
import org.dom4j.Node;
import org.orbeon.oxf.common.OXFException;
import org.orbeon.oxf.pipeline.api.PipelineContext;
import org.orbeon.oxf.processor.*;
import org.orbeon.oxf.util.ISODateUtils;
import org.orbeon.oxf.util.NumberUtils;
import org.orbeon.oxf.xml.ContentHandlerHelper;
import org.orbeon.oxf.xml.XPathUtils;
import org.orbeon.oxf.xml.dom4j.LocationData;
import org.orbeon.oxf.resources.URLFactory;
import org.orbeon.oxf.resources.ResourceManagerWrapper;
import org.xml.sax.ContentHandler;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URLConnection;
import java.net.URL;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;

/**
 * This processor scans a filesystem hierarchy and retrieves information about each file. The
 * implementation is based on a modified version of the ant directory scanner.
 *
 * Ideas for improvements:
 *
 * o componentize so that file identification and enhanced file information (such as images) can be
 *   plugged
 *
 * o you could imagine, for XML files, to extract content with XPath expressions
 */
public class DirectoryScannerProcessor extends ProcessorImpl {

    public static final String DIRECTORY_GENERATOR_NAMESPACE_URI = "http://www.orbeon.org/oxf/directory-generator";

    private static final String DIRECTORY_ELEMENT = "directory";
    private static final String FILE_ELEMENT = "file";

    private static final String EXIF_ELEMENT = "exif-info";
    private static final String IPTC_ELEMENT = "iptc-info";
    private static final String TAG_ELEMENT = "param";

    private static final boolean DEFAULT_CASE_SENSITIVE = true;
    private static final boolean DEFAULT_DEFAULT_EXCLUDES = false;

    private static final boolean DEFAULT_BASIC_INFO = false;
    private static final boolean DEFAULT_EXIF_INFO = false;
    private static final boolean DEFAULT_IPTC_INFO = false;


    public DirectoryScannerProcessor() {
        addInputInfo(new ProcessorInputOutputInfo(INPUT_CONFIG, DIRECTORY_GENERATOR_NAMESPACE_URI));
        addOutputInfo(new ProcessorInputOutputInfo(OUTPUT_DATA));
    }

    public ProcessorOutput createOutput(String name) {
        ProcessorOutput output = new ProcessorImpl.ProcessorOutputImpl(getClass(), name) {
            public void readImpl(PipelineContext context, ContentHandler contentHandler) {

                // Read config
                final Config config = (Config) readCacheInputAsObject(context, getInputByName(INPUT_CONFIG), new CacheableInputReader() {
                    public Object read(PipelineContext context, ProcessorInput input) {

                        final Document configNode = readInputAsDOM4J(context, input);
                        final Config config = new Config();

                        final String baseDirectoryURLString = XPathUtils.selectStringValueNormalize(configNode, "/config/base-directory").trim();

                        // Use location data if present so that relative URLs can be supported
                        final LocationData locationData = getLocationData();
                        final URL fullURL;
                        final String realPath;
                        try {
                            fullURL = (locationData != null && locationData.getSystemID() != null)
                                                            ? URLFactory.createURL(locationData.getSystemID(), baseDirectoryURLString)
                                                            : URLFactory.createURL(baseDirectoryURLString);

                            if (fullURL.getProtocol().equals("oxf")) {
                                // Get real path to resource path if possible
                                realPath = ResourceManagerWrapper.instance().getRealPath(fullURL.getFile());
                                if (realPath == null)
                                    throw new OXFException("Directory Scanner processor is unable to obtain the real path of the file using the oxf: protocol for the base-directory property: " + baseDirectoryURLString);
                            } else if (fullURL.getProtocol().equals("file")) {
                                String host = fullURL.getHost();
                                realPath = host + (host.length() > 0 ? ":" : "") + fullURL.getFile();
                            } else {
                                throw new OXFException("Directory Scanner processor only supports the file: and oxf: protocols for the base-directory property: " + baseDirectoryURLString);
                            }

                        } catch (MalformedURLException e) {
                            throw new OXFException(e);
                        }
                        config.setBaseDirectory(realPath);

                        for (Iterator i = XPathUtils.selectIterator(configNode, "/config/include"); i.hasNext();) {
                            final Node node = (Node) i.next();
                            final String value = XPathUtils.selectStringValueNormalize(node, ".");
                            if (value != null)
                                config.addInclude(value);
                        }
                        for (Iterator i = XPathUtils.selectIterator(configNode, "/config/exclude"); i.hasNext();) {
                            final Node node = (Node) i.next();
                            final String value = XPathUtils.selectStringValueNormalize(node, ".");
                            if (value != null)
                                config.addExclude(value);
                        }
                        final boolean caseSensitive = ProcessorUtils.selectBooleanValue(configNode, "/config/case-sensitive", DEFAULT_CASE_SENSITIVE);
                        config.setCaseSensitive(caseSensitive);
                        final boolean defaultExcludes = ProcessorUtils.selectBooleanValue(configNode, "/config/default-excludes", DEFAULT_DEFAULT_EXCLUDES);
                        config.setDefaultExcludes(defaultExcludes);

                        final boolean basicInfo = ProcessorUtils.selectBooleanValue(configNode, "/config/image-metadata/basic-info", DEFAULT_BASIC_INFO);
{'fraction_non_alphanumeric': 0.07737036530305104, 'fraction_numerical': 0.0018470378984813244, 'mean_word_length': 2.327794218074209, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5014339', 'n_tokens_mistral': 3640, 'n_tokens_neox': 3473, 'n_words': 1013}