<commit_before>/*
 * Copyright (C) 2012 B3Partners B.V.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package nl.b3p.geotools.data.arcgis;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.*;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
//import org.codehaus.httpcache4j.cache.HTTPCache;
import org.geotools.data.ows.HTTPClient;
import org.geotools.data.ows.HTTPResponse;
import org.geotools.data.ows.SimpleHttpClient;
import org.geotools.data.store.ContentDataStore;
import org.geotools.data.store.ContentEntry;
import org.geotools.data.store.ContentFeatureSource;
import org.geotools.feature.NameImpl;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.opengis.feature.type.Name;
import org.opengis.referencing.crs.CoordinateReferenceSystem;

/**
 * DataStore for ArcGIS Server REST API for MapServer or FeatureServer (read-only) services. 
 * <p>
 * Because of the nature of the REST API, paging is not suited for stateless 
 * code such as web pages. When you request a FeatureReader with a Query with
 * paging parameters set using {@link org.geotools.data.Query#setStartIndex} and
 * {@link org.geotools.data.Query#setMaxFeatures}, the data store must always
 * request all object ids matching the rest of the query and then request the
 * features in the page using the object ids. If the feature collection is very
 * large this can take a long time.
 * <p>
 * It is possible to request the object ids matching the Query using {@link ArcGISFeatureReader#getObjectIds()}
 * and use this List later on to request features with {@link ArcGISFeatureReader#getFeaturesByObjectIds(java.util.List)}.
 * The list with object ids can be cached where appropriate and be used to request a 
 * sublist for paging.
 * <p>
 * Unfortunately the object ids cannot be used as a row id in a query to pass on 
 * the start index and max features. While sometimes the object ids start at 1 
 * and increment for each subsequent feature, this is not guaranteed.
 * <p>
 * Although ArcGIS server sends ETag headers in response to requests, there is
 * not really a performance improvement using conditional HTTP requests - with
 * very large feature collections the performance is abysmal no matter what. 
 * Because of this the HTTP cache (using {@link <a href=http://httpcache4j.codehaus.org/">HttpCache4j</a>})
 * is disabled by default so the HttpCache4j libraries are not required. To enable
 * HTTP caching you must patch HttpCache4j to support non-conformant ETag headers
 * sent by ESRI (see {@link CachingHTTPClient}), rename CachingHTTPClient.java.disabled
 * and uncomment the relevant lines in this class and {@link ArcGISDataStoreFactory}. 
 * Then pass a HTTPCache instance to the constructor or using the HTTP_CACHE
 * Param for the factory. 
 * <p>
 * When reading features ArcGIS server may produce invalid JSON when it apparently 
 * has an invalid coordinate and sends "*****************" instead. Like non-well
 * formed XML we do not accept this so ArcGISFeatureReader.next() may throw an exception 
 * (consistently, the ESRI WFS server also produces invalid posLists this way).
 * <p>
 * The standard ESRI spatial querying restrictions apply: only one spatial operator
 * with the default geometry and a literal geometry operand is supported and can 
 * only be combined with other attribute queries in a Boolean AND.
 *
 * @author <NAME>
 */
public class ArcGISDataStore extends ContentDataStore {
    private static final Log log = LogFactory.getLog(ArcGISDataStore.class);
    
    private URL url;
    
    private enum ServerType {
        MapServer, FeatureServer;
    }
       
    private ServerType serverType;
    
    private CoordinateReferenceSystem crs;    

    private List<Name> typeNames = null;

    private Map<String,JSONObject> layersById = new HashMap<String,JSONObject>();

    private HTTPClient client;
    
    private String currentVersion;
    private Integer currentMajorVersion;
    
    public ArcGISDataStore(URL url) {
        this(url, null, null, null, null, null, null);
    }

    public ArcGISDataStore(URL url, String user, String passwd, Integer timeout, Boolean gzip, CoordinateReferenceSystem crs, Object httpCache) {
        this(url, user, passwd, timeout, gzip, crs, httpCache, null);
    }
    
    public ArcGISDataStore(URL url, String user, String passwd, Integer timeout, Boolean gzip, CoordinateReferenceSystem crs, Object httpCache, String currentVersion) {
        this.url = url;
        this.crs = crs;    
        this.currentVersion = currentVersion;
        if(currentVersion != null) {
            try {
                currentMajorVersion = Integer.parseInt(currentVersion.split("\\.")[0]);            
            } catch(Exception e) {
                log.warn(String.format(
                        "Invalid currentVersion specified to ArcGISDataStore: \"%s\", will ask server instead",
                        currentVersion));
                this.currentVersion = null;
            }
        }
        
        String urlString = url.toString();
        
        if(urlString.endsWith("/MapServer")) {
            serverType = ServerType.MapServer;
        } else if(urlString.endsWith("/FeatureServer")) {
            serverType = ServerType.FeatureServer;
        } else {
            throw new IllegalArgumentException("URL must end in \"/MapServer\" or \"/FeatureServer\"");
        }
        if(urlString.indexOf("/rest/") == -1) {
            throw new IllegalArgumentException("URL must contain \"/rest/\"");
        }         
        
        // Uncomment to enable HttpCache4j
        //if(httpCache != null) {
        //    client = new CachingHTTPClient((HTTPCache)httpCache);
        //    if(!Boolean.FALSE.equals(gzip)) {
        //        ((CachingHTTPClient)client).setGzip(true);
        //    }
        //} else {
            client = new SimpleHttpClient();
        //}

        client.setUser(user);
        client.setPassword(<PASSWORD>);
        if(timeout != null) {
            client.setConnectTimeout(timeout);
            client.setReadTimeout(timeout);
        }
    }
    
    public String getCurrentVersion() throws IOException {
        if(currentVersion == null) {
            log.debug("Determining currentVersion of ArcGIS service to check for 9.x (to prevent this request, provide CURRENT_VERSION parameter to DataStore)");
            
            // currentVersion not included in MapServer/ JSON in 9.3.1, get it
            // from the root services JSON

            URL originalUrl = this.url;
            
            try {
                int i = originalUrl.toString().indexOf("/rest/services");
                String servicesUrl = originalUrl.toString().substring(0, i) + "/rest/services";
                this.url = new URL(servicesUrl);
                JSONObject servicesInfo = getServerJSONResponse("?f=json");
                currentVersion = (String)servicesInfo.get("currentVersion");
                currentMajorVersion = Integer.parseInt(currentVersion.split("\\.")[0]);            
            } catch(Exception e) {
                throw new IOException("Error finding out the currentVersion of ArcGIS REST service at " + url.toString(), e);
            } finally {
                this.url = originalUrl;
            }
        }            
        return currentVersion;
    }
    
    public int getCurrentMajorVersion() throws IOException {
        if(currentVersion == null) {
            getCurrentVersion();<commit_msg>Overijssel ArcGis 10.01 server returns a (Long)1001 as version number.<commit_after>/*
 * Copyright (C) 2012 B3Partners B.V.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package nl.b3p.geotools.data.arcgis;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.*;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
//import org.codehaus.httpcache4j.cache.HTTPCache;
import org.geotools.data.ows.HTTPClient;
import org.geotools.data.ows.HTTPResponse;
import org.geotools.data.ows.SimpleHttpClient;
import org.geotools.data.store.ContentDataStore;
import org.geotools.data.store.ContentEntry;
import org.geotools.data.store.ContentFeatureSource;
import org.geotools.feature.NameImpl;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.opengis.feature.type.Name;
import org.opengis.referencing.crs.CoordinateReferenceSystem;

/**
 * DataStore for ArcGIS Server REST API for MapServer or FeatureServer (read-only) services. 
 * <p>
 * Because of the nature of the REST API, paging is not suited for stateless 
 * code such as web pages. When you request a FeatureReader with a Query with
 * paging parameters set using {@link org.geotools.data.Query#setStartIndex} and
 * {@link org.geotools.data.Query#setMaxFeatures}, the data store must always
 * request all object ids matching the rest of the query and then request the
 * features in the page using the object ids. If the feature collection is very
 * large this can take a long time.
 * <p>
 * It is possible to request the object ids matching the Query using {@link ArcGISFeatureReader#getObjectIds()}
 * and use this List later on to request features with {@link ArcGISFeatureReader#getFeaturesByObjectIds(java.util.List)}.
 * The list with object ids can be cached where appropriate and be used to request a 
 * sublist for paging.
 * <p>
 * Unfortunately the object ids cannot be used as a row id in a query to pass on 
 * the start index and max features. While sometimes the object ids start at 1 
 * and increment for each subsequent feature, this is not guaranteed.
 * <p>
 * Although ArcGIS server sends ETag headers in response to requests, there is
 * not really a performance improvement using conditional HTTP requests - with
 * very large feature collections the performance is abysmal no matter what. 
 * Because of this the HTTP cache (using {@link <a href=http://httpcache4j.codehaus.org/">HttpCache4j</a>})
 * is disabled by default so the HttpCache4j libraries are not required. To enable
 * HTTP caching you must patch HttpCache4j to support non-conformant ETag headers
 * sent by ESRI (see {@link CachingHTTPClient}), rename CachingHTTPClient.java.disabled
 * and uncomment the relevant lines in this class and {@link ArcGISDataStoreFactory}. 
 * Then pass a HTTPCache instance to the constructor or using the HTTP_CACHE
 * Param for the factory. 
 * <p>
 * When reading features ArcGIS server may produce invalid JSON when it apparently 
 * has an invalid coordinate and sends "*****************" instead. Like non-well
 * formed XML we do not accept this so ArcGISFeatureReader.next() may throw an exception 
 * (consistently, the ESRI WFS server also produces invalid posLists this way).
 * <p>
 * The standard ESRI spatial querying restrictions apply: only one spatial operator
 * with the default geometry and a literal geometry operand is supported and can 
 * only be combined with other attribute queries in a Boolean AND.
 *
 * @author <NAME>
 */
public class ArcGISDataStore extends ContentDataStore {
    private static final Log log = LogFactory.getLog(ArcGISDataStore.class);
    
    private URL url;
    
    private enum ServerType {
        MapServer, FeatureServer;
    }
       
    private ServerType serverType;
    
    private CoordinateReferenceSystem crs;    

    private List<Name> typeNames = null;

    private Map<String,JSONObject> layersById = new HashMap<String,JSONObject>();

    private HTTPClient client;
    
    private String currentVersion;
    private Integer currentMajorVersion;
    
    public ArcGISDataStore(URL url) {
        this(url, null, null, null, null, null, null);
    }

    public ArcGISDataStore(URL url, String user, String passwd, Integer timeout, Boolean gzip, CoordinateReferenceSystem crs, Object httpCache) {
        this(url, user, passwd, timeout, gzip, crs, httpCache, null);
    }
    
    public ArcGISDataStore(URL url, String user, String passwd, Integer timeout, Boolean gzip, CoordinateReferenceSystem crs, Object httpCache, String currentVersion) {
        this.url = url;
        this.crs = crs;    
        this.currentVersion = currentVersion;
        if(currentVersion != null) {
            try {
                currentMajorVersion = Integer.parseInt(currentVersion.split("\\.")[0]);            
            } catch(Exception e) {
                log.warn(String.format(
                        "Invalid currentVersion specified to ArcGISDataStore: \"%s\", will ask server instead",
                        currentVersion));
                this.currentVersion = null;
            }
        }
        
        String urlString = url.toString();
        
        if(urlString.endsWith("/MapServer")) {
            serverType = ServerType.MapServer;
        } else if(urlString.endsWith("/FeatureServer")) {
            serverType = ServerType.FeatureServer;
        } else {
            throw new IllegalArgumentException("URL must end in \"/MapServer\" or \"/FeatureServer\"");
        }
        if(urlString.indexOf("/rest/") == -1) {
            throw new IllegalArgumentException("URL must contain \"/rest/\"");
        }         
        
        // Uncomment to enable HttpCache4j
        //if(httpCache != null) {
        //    client = new CachingHTTPClient((HTTPCache)httpCache);
        //    if(!Boolean.FALSE.equals(gzip)) {
        //        ((CachingHTTPClient)client).setGzip(true);
        //    }
        //} else {
            client = new SimpleHttpClient();
        //}

        client.setUser(user);
        client.setPassword(<PASSWORD>);
        if(timeout != null) {
            client.setConnectTimeout(timeout);
            client.setReadTimeout(timeout);
        }
    }
    
    public String getCurrentVersion() throws IOException {
        if(currentVersion == null) {
            log.debug("Determining currentVersion of ArcGIS service to check for 9.x (to prevent this request, provide CURRENT_VERSION parameter to DataStore)");
            
            // currentVersion not included in MapServer/ JSON in 9.3.1, get it
            // from the root services JSON

            URL originalUrl = this.url;
            
            try {
                int i = originalUrl.toString().indexOf("/rest/services");
                String servicesUrl = originalUrl.toString().substring(0, i) + "/rest/services";
                this.url = new URL(servicesUrl);
                JSONObject servicesInfo = getServerJSONResponse("?f=json");
                Object vObject = servicesInfo.get("currentVersion");
                if (vObject instanceof String){
                    currentVersion=(String)vObject;
                //if 10.01 the version number is a long with value: 1001
                }else if (vObject instanceof Long && (Long)vObject == 1001){
                    currentVersion="10.01";
                }else{
                    currentVersion=vObject.toString();
                }
                currentMajorVersion = Integer.parseInt(currentVersion.split("\\.")[0]);            
            } catch(Exception e) {
                throw new IOException("Error finding out the currentVersion of ArcGIS REST service at " + url.toString(), e);
            } finally {
                this.url = originalUrl;
            }
        }            
        return currentVersion;
    }
    
    public int getCurrentMajorVersion() throws IOException {
        if(currentVersion == null) {
            getCurrentVersion();
{'fraction_non_alphanumeric': 0.08670756646216769, 'fraction_numerical': 0.0039731229915279, 'mean_word_length': 2.7993340732519423, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3516779', 'n_tokens_mistral': 4513, 'n_tokens_neox': 4367, 'n_words': 1589}