<commit_before>//$HeadURL$
/*----------------    FILE HEADER  ------------------------------------------
 This file is part of deegree.
 Copyright (C) 2001-2009 by:
 Department of Geography, University of Bonn
 http://www.giub.uni-bonn.de/deegree/
 lat/lon GmbH
 http://www.lat-lon.de

 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
 version 2.1 of the License, or (at your option) any later version.
 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 Lesser General Public License for more details.
 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 Contact:

 <NAME>
 lat/lon GmbH
 Aennchenstr. 19
 53177 Bonn
 Germany
 E-Mail: <EMAIL>

 Prof. Dr. <NAME>
 Department of Geography
 University of Bonn
 Meckenheimer Allee 166
 53115 Bonn
 Germany
 E-Mail: <EMAIL>
 ---------------------------------------------------------------------------*/

package org.deegree.coverage.raster.utils;

import static org.deegree.coverage.raster.utils.RasterFactory.loadRasterFromFile;

import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import javax.xml.bind.JAXBException;

import org.deegree.commons.utils.FileUtils;
import org.deegree.commons.xml.XMLAdapter;
import org.deegree.commons.xml.jaxb.JAXBUtils;
import org.deegree.coverage.AbstractCoverage;
import org.deegree.coverage.persistence.CoverageBuilder;
import org.deegree.coverage.raster.AbstractRaster;
import org.deegree.coverage.raster.MultiResolutionRaster;
import org.deegree.coverage.raster.SimpleRaster;
import org.deegree.coverage.raster.TiledRaster;
import org.deegree.coverage.raster.container.DiskBasedTileContainer;
import org.deegree.coverage.raster.container.GriddedBlobTileContainer;
import org.deegree.coverage.raster.container.IndexedMemoryTileContainer;
import org.deegree.coverage.raster.geom.RasterGeoReference;
import org.deegree.coverage.raster.geom.RasterGeoReference.OriginLocation;
import org.deegree.coverage.raster.io.RasterIOOptions;
import org.deegree.coverage.raster.io.jaxb.AbstractRasterType;
import org.deegree.coverage.raster.io.jaxb.AbstractRasterType.RasterDirectory;
import org.deegree.coverage.raster.io.jaxb.MultiResolutionRasterConfig;
import org.deegree.coverage.raster.io.jaxb.MultiResolutionRasterConfig.Resolution;
import org.deegree.coverage.raster.io.jaxb.RasterConfig;
import org.deegree.cs.coordinatesystems.ICRS;
import org.deegree.cs.persistence.CRSManager;
import org.deegree.geometry.Envelope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * The <code>RasterBuilder</code> recursively enters a given directory and creates a {@link TiledRaster} from found
 * image files.
 * 
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 * @author last edited by: $Author$
 * @version $Revision$, $Date$
 * 
 */
public class RasterBuilder implements CoverageBuilder {

    private static final String CONFIG_NS = "http://www.deegree.org/datasource/coverage/raster";

    private static final String CONFIG_JAXB_PACKAGE = "org.deegree.coverage.raster.io.jaxb";

    private static final String CONFIG_SCHEMA = "/META-INF/schemas/datasource/coverage/raster/3.0.0/raster.xsd";

    private final static Logger LOG = LoggerFactory.getLogger( RasterBuilder.class );

    /**
     * Create a {@link MultiResolutionRaster} with the origin or the world coordinate of each raster file, defined by
     * the given {@link OriginLocation}
     * 
     * @param resolutionDirectory
     *            locating the different resolutions
     * @param recursive
     *            if the sub directories of the resolution directories should be scanned as well
     * @param options
     *            containing information on the loading of the raster data.
     * @return a {@link MultiResolutionRaster} filled with {@link TiledRaster}s or <code>null</code> if the
     *         resolutionDirectory is not a directory.
     */
    public MultiResolutionRaster buildMultiResolutionRaster( File resolutionDirectory, boolean recursive,
                                                             RasterIOOptions options ) {
        if ( !resolutionDirectory.isDirectory() ) {
            return null;
        }
        return buildMultiResolutionRaster( findResolutionDirs( resolutionDirectory ), recursive, options );
    }

    /**
     * Scan the given directory for top level directories ending with a resolution.
     * 
     * @param toplevelDir
     * @return a list of directories which can be used for the building of {@link TiledRaster}s.
     */
    private final static List<File> findResolutionDirs( File toplevelDir ) {
        List<File> result = new LinkedList<File>();
        for ( File f : toplevelDir.listFiles() ) {
            if ( f.isDirectory() ) {
                double res = RasterBuilder.getPixelResolution( null, f );
                if ( !Double.isNaN( res ) ) {
                    result.add( f );
                } else {
                    LOG.info( "Directory: "
                              + f.getAbsolutePath()
                              + " can not be added to a Multiresolution raster, because it does not denote a resolution." );
                }
            }
        }
        return result;
    }

    /**
     * @param resolutionDirectories
     *            locating the different resolutions
     * @param recursive
     *            if the sub directories of the resolution directories should be scanned as well
     * @param options
     *            containing values for the loading of the raster data.
     * @return a {@link MultiResolutionRaster} filled with {@link TiledRaster}s
     */
    private MultiResolutionRaster buildMultiResolutionRaster( List<File> resolutionDirectories, boolean recursive,
                                                              RasterIOOptions options ) {
        MultiResolutionRaster mrr = new MultiResolutionRaster();
        for ( File resDir : resolutionDirectories ) {
            if ( resDir != null && resDir.isDirectory() ) {
                AbstractRaster rasterLevel = buildTiledRaster( resDir, recursive, options );
                if ( rasterLevel != null ) {
                    mrr.addRaster( rasterLevel );
                }
            }
        }
        return mrr;
    }

    /*
     * (non-Javadoc)
     * 
     * @see org.deegree.coverage.raster.utils.CoverageBuilder#getConfigNamespace()
     */
    public String getConfigNamespace() {
        return CONFIG_NS;
    }

    /*
     * (non-Javadoc)
     * 
     * @see org.deegree.coverage.raster.utils.CoverageBuilder#buildCoverage(java.net.URL)
     */
    public AbstractCoverage buildCoverage( URL configURL )
                            throws IOException {
        try {
            Object config = JAXBUtils.unmarshall( CONFIG_JAXB_PACKAGE, CONFIG_SCHEMA, configURL );

            XMLAdapter resolver = new XMLAdapter();
            resolver.setSystemId( configURL.toString() );

            if ( config instanceof MultiResolutionRasterConfig ) {
                return fromJAXB( (MultiResolutionRasterConfig) config, resolver, null );
            }
            if ( config instanceof RasterConfig ) {
                return fromJAXB( (RasterConfig) config, resolver, null, null );
            }
            LOG.warn( "An unknown object '{}' came out of JAXB parsing. This is probably a bug.", config.getClass() );
        } catch ( JAXBException e ) {
            LOG.warn( "Coverage datastore configuration from '{}' could not be read: '{}'.", configURL,
                      e.getLocalizedMessage() );
            LOG.trace( "Stack trace:", e );
        }
        return null;
    }

    /**
     * @param mrrConfig
     * @param adapter
     * @return a corresponding raster
     */
    private MultiResolutionRaster fromJAXB( MultiResolutionRasterConfig mrrConfig, XMLAdapter adapter,
                                            ICRS parentCrs ) {
        if ( mrrConfig != null ) {
            String defCRS = mrrConfig.getStorageCRS();
            ICRS crs = null;
            if ( defCRS != null ) {
                crs = CRSManager.getCRSRef( defCRS );
            }
            if ( crs == null ) {
                LOG.debug( "Using parent crs." );
                crs = parentCrs;
            }
            RasterIOOptions options = getOptions( mrrConfig, parentCrs );
            MultiResolutionRaster mrr = new MultiResolutionRaster();
            mrr.setCoordinateSystem( crs );
            for ( Resolution resolution : mrrConfig.getResolution() ) {
                if ( resolution != null ) {
                    AbstractRaster rasterLevel = fromJAXB( resolution, adapter, options, crs );
                    mrr.addRaster( rasterLevel );
                }
            }
            return mrr;
        }
        throw new NullPointerException( "The configured multi resolution raster may not be null." );
    }

    private RasterIOOptions getOptions( MultiResolutionRasterConfig config, ICRS parentCrs ) {
        RasterIOOptions opts = new RasterIOOptions();
        if ( config.getStorageCRS() != null ) {
            opts.add( "CRS", config.getStorageCRS() );
        } else {
            opts.add( "CRS", parentCrs.getAlias() );
        }
        return opts;
    }

    /**
     * @param config
     * @param adapter
     * @return a corresponding raster, null if files could not be fund
     */
    private AbstractRaster fromJAXB( AbstractRasterType config, XMLAdapter adapter, RasterIOOptions options,
                                     ICRS parentCrs ) {
        if ( config != null ) {
            String defCRS = config.getStorageCRS();
            ICRS crs = null;
            if ( defCRS != null ) {
                crs = CRSManager.getCRSRef( defCRS );
            }
            if ( crs == null ) {
                LOG.debug( "Using parent CRS, since no crs was defined." );
                crs = parentCrs;
            }
            RasterDirectory directory = config.getRasterDirectory();
            String file = config.getRasterFile();
            try {
                RasterIOOptions rOptions = new RasterIOOptions();
                rOptions.copyOf( options );
                if ( config.getOriginLocation() != null ) {
                    rOptions.add( RasterIOOptions.GEO_ORIGIN_LOCATION,
                                  config.getOriginLocation().toString().toUpperCase() );
                }
                if ( directory != null ) {
                    File rasterFiles = new File( adapter.resolve( directory.getValue().trim() ).toURI() );
                    boolean recursive = directory.isRecursive() == null ? false : directory.isRecursive();
                    if ( crs != null ) {
                        rOptions.add( RasterIOOptions.CRS, crs.getAlias() );
                    }
                    return buildTiledRaster( rasterFiles, recursive, rOptions );
                }
                if ( file != null ) {
                    file = file.trim();
                    final File loc = new File( adapter.resolve( file ).toURI() );
                    if ( !loc.exists() ) {
                        LOG.warn( "Given raster file location does not exist: " + loc.getAbsolutePath() );
                        return null;
                    }
                    rOptions.add( RasterIOOptions.OPT_FORMAT, file.substring( file.lastIndexOf( '.' ) + 1 ) );
                    AbstractRaster raster = loadRasterFromFile( loc, rOptions );
                    if ( raster != null ) {
                        raster.setCoordinateSystem( crs );
                    }
                    return raster;
                }
            } catch ( MalformedURLException e ) {
                if ( directory != null ) {
                    LOG.warn( "Could not resolve the file {}, corresponding data will not be available.",
                              directory.getValue() );
                } else {
                    LOG.warn( "Could not resolve the file {}, corresponding data will not be available.", file );
                }
            } catch ( IOException e ) {
                LOG.warn( "Could not load the file {}, corresponding data will not be available: {}", file,
                          e.getLocalizedMessage() );<commit_msg>adapted to changes in raster.xsd<commit_after>//$HeadURL$
/*----------------    FILE HEADER  ------------------------------------------
 This file is part of deegree.
 Copyright (C) 2001-2009 by:
 Department of Geography, University of Bonn
 http://www.giub.uni-bonn.de/deegree/
 lat/lon GmbH
 http://www.lat-lon.de

 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
 version 2.1 of the License, or (at your option) any later version.
 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 Lesser General Public License for more details.
 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 Contact:

 <NAME>
 lat/lon GmbH
 Aennchenstr. 19
 53177 Bonn
 Germany
 E-Mail: <EMAIL>

 Prof. Dr. <NAME>
 Department of Geography
 University of Bonn
 Meckenheimer Allee 166
 53115 Bonn
 Germany
 E-Mail: <EMAIL>
 ---------------------------------------------------------------------------*/

package org.deegree.coverage.raster.utils;

import static org.deegree.coverage.raster.utils.RasterFactory.loadRasterFromFile;

import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import javax.xml.bind.JAXBException;

import org.deegree.commons.utils.FileUtils;
import org.deegree.commons.xml.XMLAdapter;
import org.deegree.commons.xml.jaxb.JAXBUtils;
import org.deegree.coverage.AbstractCoverage;
import org.deegree.coverage.persistence.CoverageBuilder;
import org.deegree.coverage.raster.AbstractRaster;
import org.deegree.coverage.raster.MultiResolutionRaster;
import org.deegree.coverage.raster.SimpleRaster;
import org.deegree.coverage.raster.TiledRaster;
import org.deegree.coverage.raster.container.DiskBasedTileContainer;
import org.deegree.coverage.raster.container.GriddedBlobTileContainer;
import org.deegree.coverage.raster.container.IndexedMemoryTileContainer;
import org.deegree.coverage.raster.geom.RasterGeoReference;
import org.deegree.coverage.raster.geom.RasterGeoReference.OriginLocation;
import org.deegree.coverage.raster.io.RasterIOOptions;
import org.deegree.coverage.raster.io.jaxb.AbstractRasterType;
import org.deegree.coverage.raster.io.jaxb.AbstractRasterType.RasterDirectory;
import org.deegree.coverage.raster.io.jaxb.MultiResolutionRasterConfig;
import org.deegree.coverage.raster.io.jaxb.MultiResolutionRasterConfig.Resolution;
import org.deegree.coverage.raster.io.jaxb.RasterConfig;
import org.deegree.cs.coordinatesystems.ICRS;
import org.deegree.cs.persistence.CRSManager;
import org.deegree.geometry.Envelope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * The <code>RasterBuilder</code> recursively enters a given directory and creates a {@link TiledRaster} from found
 * image files.
 * 
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 * @author last edited by: $Author$
 * @version $Revision$, $Date$
 * 
 */
public class RasterBuilder implements CoverageBuilder {

    private static final String CONFIG_NS = "http://www.deegree.org/datasource/coverage/raster";

    private static final String CONFIG_JAXB_PACKAGE = "org.deegree.coverage.raster.io.jaxb";

    private static final String CONFIG_SCHEMA = "/META-INF/schemas/datasource/coverage/raster/3.0.0/raster.xsd";

    private final static Logger LOG = LoggerFactory.getLogger( RasterBuilder.class );

    /**
     * Create a {@link MultiResolutionRaster} with the origin or the world coordinate of each raster file, defined by
     * the given {@link OriginLocation}
     * 
     * @param resolutionDirectory
     *            locating the different resolutions
     * @param recursive
     *            if the sub directories of the resolution directories should be scanned as well
     * @param options
     *            containing information on the loading of the raster data.
     * @return a {@link MultiResolutionRaster} filled with {@link TiledRaster}s or <code>null</code> if the
     *         resolutionDirectory is not a directory.
     */
    public MultiResolutionRaster buildMultiResolutionRaster( File resolutionDirectory, boolean recursive,
                                                             RasterIOOptions options ) {
        if ( !resolutionDirectory.isDirectory() ) {
            return null;
        }
        return buildMultiResolutionRaster( findResolutionDirs( resolutionDirectory ), recursive, options );
    }

    /**
     * Scan the given directory for top level directories ending with a resolution.
     * 
     * @param toplevelDir
     * @return a list of directories which can be used for the building of {@link TiledRaster}s.
     */
    private final static List<File> findResolutionDirs( File toplevelDir ) {
        List<File> result = new LinkedList<File>();
        for ( File f : toplevelDir.listFiles() ) {
            if ( f.isDirectory() ) {
                double res = RasterBuilder.getPixelResolution( null, f );
                if ( !Double.isNaN( res ) ) {
                    result.add( f );
                } else {
                    LOG.info( "Directory: "
                              + f.getAbsolutePath()
                              + " can not be added to a Multiresolution raster, because it does not denote a resolution." );
                }
            }
        }
        return result;
    }

    /**
     * @param resolutionDirectories
     *            locating the different resolutions
     * @param recursive
     *            if the sub directories of the resolution directories should be scanned as well
     * @param options
     *            containing values for the loading of the raster data.
     * @return a {@link MultiResolutionRaster} filled with {@link TiledRaster}s
     */
    private MultiResolutionRaster buildMultiResolutionRaster( List<File> resolutionDirectories, boolean recursive,
                                                              RasterIOOptions options ) {
        MultiResolutionRaster mrr = new MultiResolutionRaster();
        for ( File resDir : resolutionDirectories ) {
            if ( resDir != null && resDir.isDirectory() ) {
                AbstractRaster rasterLevel = buildTiledRaster( resDir, recursive, options );
                if ( rasterLevel != null ) {
                    mrr.addRaster( rasterLevel );
                }
            }
        }
        return mrr;
    }

    /*
     * (non-Javadoc)
     * 
     * @see org.deegree.coverage.raster.utils.CoverageBuilder#getConfigNamespace()
     */
    public String getConfigNamespace() {
        return CONFIG_NS;
    }

    /*
     * (non-Javadoc)
     * 
     * @see org.deegree.coverage.raster.utils.CoverageBuilder#buildCoverage(java.net.URL)
     */
    public AbstractCoverage buildCoverage( URL configURL )
                            throws IOException {
        try {
            Object config = JAXBUtils.unmarshall( CONFIG_JAXB_PACKAGE, CONFIG_SCHEMA, configURL );

            XMLAdapter resolver = new XMLAdapter();
            resolver.setSystemId( configURL.toString() );

            if ( config instanceof MultiResolutionRasterConfig ) {
                return fromJAXB( (MultiResolutionRasterConfig) config, resolver, null );
            }
            if ( config instanceof RasterConfig ) {
                return fromJAXB( (RasterConfig) config, resolver, null, null );
            }
            LOG.warn( "An unknown object '{}' came out of JAXB parsing. This is probably a bug.", config.getClass() );
        } catch ( JAXBException e ) {
            LOG.warn( "Coverage datastore configuration from '{}' could not be read: '{}'.", configURL,
                      e.getLocalizedMessage() );
            LOG.trace( "Stack trace:", e );
        }
        return null;
    }

    /**
     * @param mrrConfig
     * @param adapter
     * @return a corresponding raster
     */
    private MultiResolutionRaster fromJAXB( MultiResolutionRasterConfig mrrConfig, XMLAdapter adapter,
                                            ICRS parentCrs ) {
        if ( mrrConfig != null ) {
            String defCRS = mrrConfig.getStorageCRS();
            ICRS crs = null;
            if ( defCRS != null ) {
                crs = CRSManager.getCRSRef( defCRS );
            }
            if ( crs == null ) {
                LOG.debug( "Using parent crs." );
                crs = parentCrs;
            }
            RasterIOOptions options = getOptions( mrrConfig, parentCrs );
            MultiResolutionRaster mrr = new MultiResolutionRaster();
            mrr.setCoordinateSystem( crs );
            for ( Resolution resolution : mrrConfig.getResolution() ) {
                if ( resolution != null ) {
                    AbstractRaster rasterLevel = fromJAXB( resolution, adapter, options, crs );
                    mrr.addRaster( rasterLevel );
                }
            }
            return mrr;
        }
        throw new NullPointerException( "The configured multi resolution raster may not be null." );
    }

    private RasterIOOptions getOptions( MultiResolutionRasterConfig config, ICRS parentCrs ) {
        RasterIOOptions opts = new RasterIOOptions();
        if ( config.getStorageCRS() != null ) {
            opts.add( "CRS", config.getStorageCRS() );
        } else {
            opts.add( "CRS", parentCrs.getAlias() );
        }
        return opts;
    }

    /**
     * @param config
     * @param adapter
     * @return a corresponding raster, null if files could not be fund
     */
    private AbstractRaster fromJAXB( AbstractRasterType config, XMLAdapter adapter, RasterIOOptions options,
                                     ICRS parentCrs ) {
        if ( config != null ) {
            String defCRS = config.getStorageCRS();
            ICRS crs = null;
            if ( defCRS != null ) {
                crs = CRSManager.getCRSRef( defCRS );
            }
            if ( crs == null ) {
                LOG.debug( "Using parent CRS, since no crs was defined." );
                crs = parentCrs;
            }
            RasterDirectory directory = config.getRasterDirectory();
            String file = config.getRasterFile();
            try {
                RasterIOOptions rOptions = new RasterIOOptions();
                rOptions.copyOf( options );
                if ( config.getOriginLocation() != null ) {
                    rOptions.add( RasterIOOptions.GEO_ORIGIN_LOCATION,
                                  config.getOriginLocation().toString().toUpperCase() );
                }
                if ( directory != null ) {
                    File rasterFiles = new File( adapter.resolve( directory.getValue().trim() ).toURI() );
                    boolean recursive = directory.isRecursive();
                    if ( crs != null ) {
                        rOptions.add( RasterIOOptions.CRS, crs.getAlias() );
                    }
                    return buildTiledRaster( rasterFiles, recursive, rOptions );
                }
                if ( file != null ) {
                    file = file.trim();
                    final File loc = new File( adapter.resolve( file ).toURI() );
                    if ( !loc.exists() ) {
                        LOG.warn( "Given raster file location does not exist: " + loc.getAbsolutePath() );
                        return null;
                    }
                    rOptions.add( RasterIOOptions.OPT_FORMAT, file.substring( file.lastIndexOf( '.' ) + 1 ) );
                    AbstractRaster raster = loadRasterFromFile( loc, rOptions );
                    if ( raster != null ) {
                        raster.setCoordinateSystem( crs );
                    }
                    return raster;
                }
            } catch ( MalformedURLException e ) {
                if ( directory != null ) {
                    LOG.warn( "Could not resolve the file {}, corresponding data will not be available.",
                              directory.getValue() );
                } else {
                    LOG.warn( "Could not resolve the file {}, corresponding data will not be available.", file );
                }
            } catch ( IOException e ) {
                LOG.warn( "Could not load the file {}, corresponding data will not be available: {}", file,
                          e.getLocalizedMessage() );
{'fraction_non_alphanumeric': 0.09795121951219513, 'fraction_numerical': 0.0035121951219512196, 'mean_word_length': 2.0280042538107055, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7064689', 'n_tokens_mistral': 7067, 'n_tokens_neox': 6788, 'n_words': 1855}