<commit_before>/*
   Copyright ${year}  Knowledge Media Institute - The Open University

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
package uk.ac.open.kmi.iserve.discovery.engine;

import java.io.StringWriter;
import java.net.URLDecoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import javax.ws.rs.GET;
import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriBuilder;
import javax.ws.rs.core.UriInfo;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;

import org.apache.abdera.model.Entry;
import org.apache.abdera.model.Feed;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.GetMethod;

import org.w3c.dom.DOMImplementation;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.ls.DOMImplementationLS;
import org.w3c.dom.ls.LSOutput;
import org.w3c.dom.ls.LSSerializer;

import com.sun.jersey.api.client.AsyncWebResource;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.client.apache.config.DefaultApacheHttpClientConfig;
import com.sun.jersey.client.apache.ApacheHttpClient;
import com.sun.jersey.client.apache.ApacheHttpClientHandler;

import uk.ac.open.kmi.iserve.discovery.api.DiscoveryException;
import uk.ac.open.kmi.iserve.discovery.api.util.DiscoveryUtil;

/**
 * provides base functionality for atom combinators
 * todo relative URIs in the atom feeds are currently not resolved against base
 * @author <NAME>
 *
 */
public abstract class AtomBase {

    /**
     * <p>HTTP header for sending authentication credentials.</p>
     */
    private static final String AUTHENTICATION_HEADER = "Authorization";

    /**
     * <p>HTTP authentication header value to include on each request.</p>
     */
    private String authentication = null;

	private Client httpClient;

    /**
     * <p>Default media type for content element.</p>
     */
    private static final MediaType CONTENT_MEDIA_TYPE = MediaType.APPLICATION_XML_TYPE;

    // -------------------------------------------------------- Static Variables
    /**
     * <p>Default media type for entry entities.</p>
     */
    private static final MediaType ENTRY_MEDIA_TYPE;
    static {
        Map<String,String> params = new HashMap<String,String>(1);
        params.put("type", "entry");
        ENTRY_MEDIA_TYPE = new MediaType("application", "xml", params);
    }

    /**
     * <p>Default media type for feed entities.</p>
     */
    private static final MediaType FEED_MEDIA_TYPE;

	private static final int MAX_CONNECTIONS_PER_HOST = 3;

	private static final int MAX_TOTAL_CONNECTIONS = 10;
	
    static {
        Map<String,String> params = new HashMap<String,String>(1);
        params.put("type", "feed");
        FEED_MEDIA_TYPE = new MediaType("application", "xml", params);
    }

    
    /**
     * <p>Construct a fully configured client instance.</p>
     */
    public AtomBase() {
        MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); 
        connectionManager.getParams().setDefaultMaxConnectionsPerHost(MAX_CONNECTIONS_PER_HOST); 
        connectionManager.getParams().setMaxTotalConnections(MAX_TOTAL_CONNECTIONS); 

        ApacheHttpClientHandler apacheHandler = new ApacheHttpClientHandler(
            	new HttpClient(connectionManager));
        
        apacheHandler.getConfig().getProperties().put(DefaultApacheHttpClientConfig.PROPERTY_PREEMPTIVE_AUTHENTICATION, 
                Boolean.TRUE); 

        httpClient = new ApacheHttpClient(apacheHandler);
    }

    // ------------------------------------------------- Contacts Public Methods

	@GET
	@Produces ({MediaType.APPLICATION_ATOM_XML,
		MediaType.APPLICATION_JSON,
		MediaType.TEXT_XML})
	public Response operate(@Context UriInfo ui) {
		if ( ui.getAbsolutePath().toString().contains("data/atom/") ) {
			MultivaluedMap<String, String> params = ui.getQueryParameters();
			List<String> uris = params.get("f");
			if ( uris.size() < 2 ) {
				throw new WebApplicationException(new DiscoveryException("Atom feed combinators work with at least two f parameters (as feed URIs)"), 403);
			}

			// TODO: potential for misuse: with a single request to our server, a malicious client can make our server make multiple requests to other server(s)
			// maybe limit the applicability of this atom union only to URIs on the same server?
			Feed result;
			try {
				result = atomCombination(ui.getBaseUri().toString(), uris);
			} catch (Exception e) {
				throw new WebApplicationException(e, 403);
			}

			return Response.ok(result).build();
		}
		return Response.status(404).build();
	}

	/**
	 * FIXME: This ignores that Substraction is order-dependent!
	 * 
	 * @param requestURI
	 * @param uris
	 * @return
	 * @throws Exception
	 */
	private Feed atomCombination(String requestURI, List<String> uris) throws Exception {
		
		resolveAndCheckURIs(requestURI, uris);

		// download the feeds
		List<Feed> feeds = downloadFeeds(uris);
		int feedCount = feeds.size();

		if ( feedCount == 0 ) {
			throw new DiscoveryException("No supported feeds found");
		} else if ( feedCount == 1 ) {
			return feeds.get(0);
		}
		
		//feedUpdated = extractFeedData(feeds, feedTitles, feedRights, entriesByFeed, entriesByID);
		
		Feed combinedFeed = DiscoveryUtil.getAbderaInstance().getFactory().newFeed();
		combinedFeed.setId(requestURI);
		combinedFeed.addLink(requestURI,"self");
		combinedFeed.setGenerator("http://iserve.kmi.open.ac.uk/", null, 
				"iServe Atom " + combinatorName() + " combinator 2010/06/23"); 
		
		String rights = "The constituent feeds have the following rights statements (in no particular order): ";
		
		Iterator<Feed> iterator = feeds.iterator();
		Set<Entry> combination = new HashSet(iterator.next().getEntries());
		while ( iterator.hasNext() ) {
			Feed feed = (Feed) iterator.next();
			combination = combineResults(combination, feed.getEntries());
			// FIXME: merge title etc
			// feed.setTitle(plugin.getFeedTitle());
			rights += "\"" + feed.getRights() + "\"";
			if ( iterator.hasNext() ) {
				rights += ", ";
			} else {
				rights += ".";
			}
		}

		combinedFeed.setUpdated(new Date());
		combinedFeed.setRights(rights);
		return combinedFeed;
	}

	/**
	 * Given the results in the feeds to combine, generate the combination
	 * according to the concrete Atom operator implemented
	 * 
	 * @param combination
	 * @param entries
	 * @return result of the combination
	 */
	abstract Set<Entry> combineResults(Set<Entry> combination, List<Entry> entries);

	private void resolveAndCheckURIs(String requestURI, List<String> uris) {
		for ( int i = 0; i < uris.size(); i++ ) {
			UriBuilder uriBuilder = UriBuilder.fromUri(requestURI);
			UriBuilder properUriBuilder = uriBuilder.path(uris.get(i));
			java.net.URI properuri = properUriBuilder.build("");
			String scheme = properuri.getScheme();
			if ( !"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme) ) {
				System.err.println("only http and https schemes supported; bad URI: " + properuri.toString());
				uris.remove(i);
			} else {
				uris.set(i, URLDecoder.decode(properuri.toString()));
			}
		}
	}

/*
	private Date extractFeedData(Feed[] feeds, String[] feedTitles, Set<String> feedRights,
			List<Element>[] entriesByFeed, Map<String, Element[]> entriesByID) throws ParseException {
		Date feedUpdated = null;
		for ( int i = 0; i < feeds.length; i++ ) {
			Document feed = feeds[i];
			List<Element> entries = new ArrayList<Element>();
			entriesByFeed[i] = entries;
            
			if ( feed == null ) {
				continue;
			}

			Element root = feed.getDocumentElement();
			if ( root == null ) {
				System.err.println("empty document number " + i);
				continue;
			}
			if ( !ATOM_NS.equals(root.getNamespaceURI()) || !"feed".equals(root.getLocalName()) ) {
				System.err.println("unknown root element: {" + root.getNamespaceURI() + "}" + root.getLocalName());
				continue;
			}<commit_msg>fix a bug of Atom combination<commit_after>/*
   Copyright ${year}  Knowledge Media Institute - The Open University

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
package uk.ac.open.kmi.iserve.discovery.engine;

import java.io.StringWriter;
import java.net.URLDecoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import javax.ws.rs.GET;
import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriBuilder;
import javax.ws.rs.core.UriInfo;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;

import org.apache.abdera.model.Entry;
import org.apache.abdera.model.Feed;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.GetMethod;

import org.w3c.dom.DOMImplementation;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.ls.DOMImplementationLS;
import org.w3c.dom.ls.LSOutput;
import org.w3c.dom.ls.LSSerializer;

import com.sun.jersey.api.client.AsyncWebResource;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.client.apache.config.DefaultApacheHttpClientConfig;
import com.sun.jersey.client.apache.ApacheHttpClient;
import com.sun.jersey.client.apache.ApacheHttpClientHandler;

import uk.ac.open.kmi.iserve.discovery.api.DiscoveryException;
import uk.ac.open.kmi.iserve.discovery.api.util.DiscoveryUtil;

/**
 * provides base functionality for atom combinators
 * todo relative URIs in the atom feeds are currently not resolved against base
 * @author <NAME>
 *
 */
public abstract class AtomBase {

    /**
     * <p>HTTP header for sending authentication credentials.</p>
     */
    private static final String AUTHENTICATION_HEADER = "Authorization";

    /**
     * <p>HTTP authentication header value to include on each request.</p>
     */
    private String authentication = null;

	private Client httpClient;

    /**
     * <p>Default media type for content element.</p>
     */
    private static final MediaType CONTENT_MEDIA_TYPE = MediaType.APPLICATION_XML_TYPE;

    // -------------------------------------------------------- Static Variables
    /**
     * <p>Default media type for entry entities.</p>
     */
    private static final MediaType ENTRY_MEDIA_TYPE;
    static {
        Map<String,String> params = new HashMap<String,String>(1);
        params.put("type", "entry");
        ENTRY_MEDIA_TYPE = new MediaType("application", "xml", params);
    }

    /**
     * <p>Default media type for feed entities.</p>
     */
    private static final MediaType FEED_MEDIA_TYPE;

	private static final int MAX_CONNECTIONS_PER_HOST = 3;

	private static final int MAX_TOTAL_CONNECTIONS = 10;
	
    static {
        Map<String,String> params = new HashMap<String,String>(1);
        params.put("type", "feed");
        FEED_MEDIA_TYPE = new MediaType("application", "xml", params);
    }

    
    /**
     * <p>Construct a fully configured client instance.</p>
     */
    public AtomBase() {
        MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); 
        connectionManager.getParams().setDefaultMaxConnectionsPerHost(MAX_CONNECTIONS_PER_HOST); 
        connectionManager.getParams().setMaxTotalConnections(MAX_TOTAL_CONNECTIONS); 

        ApacheHttpClientHandler apacheHandler = new ApacheHttpClientHandler(
            	new HttpClient(connectionManager));
        
        apacheHandler.getConfig().getProperties().put(DefaultApacheHttpClientConfig.PROPERTY_PREEMPTIVE_AUTHENTICATION, 
                Boolean.TRUE); 

        httpClient = new ApacheHttpClient(apacheHandler);
    }

    // ------------------------------------------------- Contacts Public Methods

	@GET
	@Produces ({MediaType.APPLICATION_ATOM_XML,
		MediaType.APPLICATION_JSON,
		MediaType.TEXT_XML})
	public Response operate(@Context UriInfo ui) {
		if ( ui.getAbsolutePath().toString().contains("data/atom/") ) {
			MultivaluedMap<String, String> params = ui.getQueryParameters();
			List<String> uris = params.get("f");
			if ( uris.size() < 2 ) {
				throw new WebApplicationException(new DiscoveryException("Atom feed combinators work with at least two f parameters (as feed URIs)"), 403);
			}

			// TODO: potential for misuse: with a single request to our server, a malicious client can make our server make multiple requests to other server(s)
			// maybe limit the applicability of this atom union only to URIs on the same server?
			Feed result;
			try {
				result = atomCombination(ui.getBaseUri().toString(), uris);
			} catch (Exception e) {
				throw new WebApplicationException(e, 403);
			}

			return Response.ok(result).build();
		}
		return Response.status(404).build();
	}

	/**
	 * FIXME: This ignores that Substraction is order-dependent!
	 * 
	 * @param requestURI
	 * @param uris
	 * @return
	 * @throws Exception
	 */
	private Feed atomCombination(String requestURI, List<String> uris) throws Exception {
		
		resolveAndCheckURIs(requestURI, uris);

		// download the feeds
		List<Feed> feeds = downloadFeeds(uris);
		int feedCount = feeds.size();

		if ( feedCount == 0 ) {
			throw new DiscoveryException("No supported feeds found");
		} else if ( feedCount == 1 ) {
			return feeds.get(0);
		}
		
		//feedUpdated = extractFeedData(feeds, feedTitles, feedRights, entriesByFeed, entriesByID);
		
		Feed combinedFeed = DiscoveryUtil.getAbderaInstance().getFactory().newFeed();
		combinedFeed.setId(requestURI);
		combinedFeed.addLink(requestURI,"self");
		combinedFeed.setGenerator("http://iserve.kmi.open.ac.uk/", null, 
				"iServe Atom " + combinatorName() + " combinator 2010/06/23"); 
		
		String rights = "The constituent feeds have the following rights statements (in no particular order): ";
		
		Iterator<Feed> iterator = feeds.iterator();
		Set<Entry> combination = new HashSet(iterator.next().getEntries());
		while ( iterator.hasNext() ) {
			Feed feed = (Feed) iterator.next();
			combination = combineResults(combination, feed.getEntries());
			// FIXME: merge title etc
			// feed.setTitle(plugin.getFeedTitle());
			rights += "\"" + feed.getRights() + "\"";
			if ( iterator.hasNext() ) {
				rights += ", ";
			} else {
				rights += ".";
			}
		}

		if ( combination != null ) {
			for ( Entry result : combination ) {
				combinedFeed.addEntry(result);
			}
		}

		combinedFeed.setUpdated(new Date());
		combinedFeed.setRights(rights);
		return combinedFeed;
	}

	/**
	 * Given the results in the feeds to combine, generate the combination
	 * according to the concrete Atom operator implemented
	 * 
	 * @param combination
	 * @param entries
	 * @return result of the combination
	 */
	abstract Set<Entry> combineResults(Set<Entry> combination, List<Entry> entries);

	private void resolveAndCheckURIs(String requestURI, List<String> uris) {
		for ( int i = 0; i < uris.size(); i++ ) {
			UriBuilder uriBuilder = UriBuilder.fromUri(requestURI);
			UriBuilder properUriBuilder = uriBuilder.path(uris.get(i));
			java.net.URI properuri = properUriBuilder.build("");
			String scheme = properuri.getScheme();
			if ( !"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme) ) {
				System.err.println("only http and https schemes supported; bad URI: " + properuri.toString());
				uris.remove(i);
			} else {
				uris.set(i, URLDecoder.decode(properuri.toString()).replaceAll("#", "%23"));
			}
		}
	}

/*
	private Date extractFeedData(Feed[] feeds, String[] feedTitles, Set<String> feedRights,
			List<Element>[] entriesByFeed, Map<String, Element[]> entriesByID) throws ParseException {
		Date feedUpdated = null;
		for ( int i = 0; i < feeds.length; i++ ) {
			Document feed = feeds[i];
			List<Element> entries = new ArrayList<Element>();
			entriesByFeed[i] = entries;
            
			if ( feed == null ) {
				continue;
			}

			Element root = feed.getDocumentElement();
			if ( root == null ) {
				System.err.println("empty document number " + i);
				continue;
			}
			if ( !ATOM_NS.equals(root.getNamespaceURI()) || !"feed".equals(root.getLocalName()) ) {
				System.err.println("unknown root element: {" + root.getNamespaceURI() + "}" + root.getLocalName());
				continue;
			}
{'fraction_non_alphanumeric': 0.12693835355568855, 'fraction_numerical': 0.0043527939496164104, 'mean_word_length': 3.66497461928934, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3145520', 'n_tokens_mistral': 5913, 'n_tokens_neox': 5340, 'n_words': 1358}