<commit_before>/*
 *  This file is part of the Wayback archival access software
 *   (http://archive-access.sourceforge.net/projects/wayback/).
 *
 *  Licensed to the Internet Archive (IA) by one or more individual 
 *  contributors. 
 *
 *  The IA licenses this file to You under the Apache License, Version 2.0
 *  (the "License"); you may not use this file except in compliance with
 *  the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
package org.archive.wayback.webapp;

import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.archive.format.gzip.zipnum.ZipNumBlockLoader;
import org.archive.wayback.ExceptionRenderer;
import org.archive.wayback.QueryRenderer;
import org.archive.wayback.ReplayDispatcher;
import org.archive.wayback.ReplayRenderer;
import org.archive.wayback.RequestParser;
import org.archive.wayback.ResultURIConverter;
import org.archive.wayback.UrlCanonicalizer;
import org.archive.wayback.accesscontrol.ExclusionFilterFactory;
import org.archive.wayback.archivalurl.ArchivalUrl;
import org.archive.wayback.core.CaptureSearchResult;
import org.archive.wayback.core.CaptureSearchResults;
import org.archive.wayback.core.Resource;
import org.archive.wayback.core.SearchResults;
import org.archive.wayback.core.UIResults;
import org.archive.wayback.core.UrlSearchResults;
import org.archive.wayback.core.WaybackRequest;
import org.archive.wayback.exception.AccessControlException;
import org.archive.wayback.exception.AdministrativeAccessControlException;
import org.archive.wayback.exception.AnchorWindowTooSmallException;
import org.archive.wayback.exception.AuthenticationControlException;
import org.archive.wayback.exception.BadQueryException;
import org.archive.wayback.exception.BaseExceptionRenderer;
import org.archive.wayback.exception.BetterRequestException;
import org.archive.wayback.exception.ConfigurationException;
import org.archive.wayback.exception.ResourceIndexNotAvailableException;
import org.archive.wayback.exception.ResourceNotAvailableException;
import org.archive.wayback.exception.ResourceNotInArchiveException;
import org.archive.wayback.exception.SpecificCaptureReplayException;
import org.archive.wayback.exception.WaybackException;
import org.archive.wayback.resourceindex.filters.ExclusionFilter;
import org.archive.wayback.resourceindex.filters.WARCRevisitAnnotationFilter;
import org.archive.wayback.resourcestore.resourcefile.WarcResource;
import org.archive.wayback.util.Timestamp;
import org.archive.wayback.util.operator.BooleanOperator;
import org.archive.wayback.util.url.UrlOperations;
import org.archive.wayback.util.webapp.AbstractRequestHandler;
import org.archive.wayback.util.webapp.ShutdownListener;
import org.archive.wayback.webapp.LiveWebRedirector.LiveWebState;

/**
 * Retains all information about a particular Wayback configuration
 * within a ServletContext, including holding references to the
 * implementation instances of the primary Wayback classes:
 * 
 * 		RequestParser
 *		ResourceIndex(via WaybackCollection)
 *		ResourceStore(via WaybackCollection)
 *		QueryRenderer
 *		ReplayDispatcher
 *		ExceptionRenderer
 *		ResultURIConverter
 *
 *
 * @author brad
 * @version $Date$, $Revision$
 */
public class AccessPoint extends AbstractRequestHandler 
implements ShutdownListener {
	/** webapp relative location of Interstitial.jsp */
	public final static String INTERSTITIAL_JSP = "jsp/Interstitial.jsp";
	/** argument for Interstitial.jsp target URL */
	public final static String INTERSTITIAL_TARGET = "target";
	/** argument for Interstitial.jsp seconds to delay */
	public final static String INTERSTITIAL_SECONDS = "seconds";

	/** argument for Interstitial.jsp msse for replay date */
	public final static String INTERSTITIAL_DATE = "date";
	/** argument for Interstitial.jsp URL being loaded */
	public final static String INTERSTITIAL_URL = "url";
	
	public final static String REVISIT_STR = "warc/revisit";
	public final static String EMPTY_VALUE = "-";
	
	public final static String RUNTIME_ERROR_HEADER = "X-Archive-Wayback-Runtime-Error";
	//public final static String NOTFOUND_ERROR_HEADER = "X-Archive-Wayback-Not-Found";

	private static final Logger LOGGER = Logger.getLogger(
			AccessPoint.class.getName());

	private boolean exactHostMatch = false;
	private boolean exactSchemeMatch = false;
	private boolean useAnchorWindow = false;
	private boolean useServerName = false;
	private boolean serveStatic = true;
	private boolean bounceToReplayPrefix = false;
	private boolean bounceToQueryPrefix = false;
	private boolean forceCleanQueries = true;

	private boolean timestampSearch = false;
	
	public static enum PerfStat
	{
		IndexQueryTotal,
		WArcResource,
		Total,
	}
	
	private String errorMsgHeader = RUNTIME_ERROR_HEADER;
	private String perfStatsHeader = "X-Archive-Wayback-Perf";
	private String warcFileHeader = "x-archive-src";
	
	private boolean enableErrorMsgHeader = false;
	private boolean enablePerfStatsHeader = false;
	private boolean enableWarcFileHeader = false;
		
	private LiveWebRedirector liveWebRedirector;
	
	private String staticPrefix = null;
	private String queryPrefix = null;
	private String replayPrefix = null;
	
	private String interstitialJsp = INTERSTITIAL_JSP;

	private String refererAuth = null;

	private Locale locale = null;

	private Properties configs = null;

	private List<String> filePatterns = null;
	private List<String> fileIncludePrefixes = null;
	private List<String> fileExcludePrefixes = null;

	private WaybackCollection  collection   = null;
	private ExceptionRenderer  exception    = new BaseExceptionRenderer();
	private QueryRenderer      query        = null;
	private RequestParser      parser       = null;
	private ReplayDispatcher   replay       = null;
	private ResultURIConverter uriConverter = null;

	private ExclusionFilterFactory exclusionFactory = null;
	private BooleanOperator<WaybackRequest> authentication = null;
	private boolean requestAuth = true;
	
	private long embargoMS = 0;
	private CustomResultFilterFactory filterFactory = null;
	
	private UrlCanonicalizer selfRedirectCanonicalizer = null;
	
	private int maxRedirectAttempts = 0;

	public void init() {
		checkAccessPointAware(collection,exception,query,parser,replay,
				uriConverter,exclusionFactory, authentication, filterFactory);
	}
	
	protected boolean dispatchLocal(HttpServletRequest httpRequest,
			HttpServletResponse httpResponse) 
	throws ServletException, IOException {
		if(LOGGER.isLoggable(Level.FINE)) {
			LOGGER.fine("Local dispatch /" + translateRequestPath(httpRequest));
		}
		if(!serveStatic) {
			return false;
		}
//		String contextRelativePath = httpRequest.getServletPath();
		String translatedNoQuery = "/" + translateRequestPath(httpRequest);
//		String absPath = getServletContext().getRealPath(contextRelativePath);
		String absPath = getServletContext().getRealPath(translatedNoQuery);
		
		//IK: added null check for absPath, it may be null (ex. on jetty)
		if (absPath != null) {
			File test = new File(absPath);
			if((test != null) && !test.exists()) {
				return false;
			}
		}
				
		String translatedQ = "/" + translateRequestPathQuery(httpRequest);

		WaybackRequest wbRequest = new WaybackRequest();
//			wbRequest.setContextPrefix(getUrlRoot());
		wbRequest.setAccessPoint(this);
		wbRequest.fixup(httpRequest);
		UIResults uiResults = new UIResults(wbRequest,uriConverter);
		try {
			uiResults.forward(httpRequest, httpResponse, translatedQ);
			return true;
		} catch(IOException e) {
			// TODO: figure out if we got IO because of a missing dispatcher
		}

		return false;
	}

	/**
	 * @param httpRequest HttpServletRequest which is being handled 
	 * @param httpResponse HttpServletResponse which is being handled 
	 * @return true if the request was actually handled
	 * @throws ServletException per usual
	 * @throws IOException per usual
	 */
	public boolean handleRequest(HttpServletRequest httpRequest,
			HttpServletResponse httpResponse) 
	throws ServletException, IOException {

		WaybackRequest wbRequest = null;
		boolean handled = false;

		try {
			PerfStats.clearAll();
			
			if (this.isEnablePerfStatsHeader() && (perfStatsHeader != null)) {
				PerfStats.timeStart(PerfStat.Total);
				httpResponse = new PerfWritingHttpServletResponse(httpResponse, PerfStat.Total, perfStatsHeader);
			}
			
			String inputPath = translateRequestPathQuery(httpRequest);
			Thread.currentThread().setName("Thread " + 
					Thread.currentThread().getId() + " " + getBeanName() + 
					" handling: " + inputPath);
			LOGGER.fine("Handling translated: " + inputPath);
			wbRequest = getParser().parse(httpRequest, this);

			if(wbRequest != null) {
				handled = true;

				// TODO: refactor this code into RequestParser implementations
				wbRequest.setAccessPoint(this);
//				wbRequest.setContextPrefix(getAbsoluteLocalPrefix(httpRequest));
//				wbRequest.setContextPrefix(getUrlRoot());
				wbRequest.fixup(httpRequest);
				// end of refactor

				if(getAuthentication() != null) {
					if(!getAuthentication().isTrue(wbRequest)) {
						throw new AuthenticationControlException(
								"Unauthorized", isRequestAuth());
					}
				}

				if(getExclusionFactory() != null) {
					ExclusionFilter exclusionFilter = 
						getExclusionFactory().get();
					if(exclusionFilter == null) {
						throw new AdministrativeAccessControlException(
								"AccessControl list unavailable");
					}
					wbRequest.setExclusionFilter(exclusionFilter);
				}
				// TODO: refactor this into RequestParser implementations, so a
				// user could alter requests to change the behavior within a
				// single AccessPoint. For now, this is a simple way to expose
				// the feature to configuration.g
				wbRequest.setExactScheme(isExactSchemeMatch());

				if(wbRequest.isReplayRequest()) {
					if(bounceToReplayPrefix) {
						// we don't accept replay requests on this AccessPoint
						// bounce the user to the right place:
						String suffix = translateRequestPathQuery(httpRequest);
						String replayUrl = replayPrefix + suffix;
						httpResponse.sendRedirect(replayUrl);
						return true;
					}
					handleReplay(wbRequest,httpRequest,httpResponse);
					
				} else {

					if(bounceToQueryPrefix) {
						// we don't accept replay requests on this AccessPoint
						// bounce the user to the right place:
						String suffix = translateRequestPathQuery(httpRequest);
						String replayUrl = queryPrefix + suffix;
						httpResponse.sendRedirect(replayUrl);
						return true;
					}
					wbRequest.setExactHost(isExactHostMatch());
					handleQuery(wbRequest,httpRequest,httpResponse);
				}
			} else {
				handled = dispatchLocal(httpRequest,httpResponse);
			}
			
		} catch(BetterRequestException e) {
			e.generateResponse(httpResponse);
			handled = true;

		} catch(WaybackException e) {
			
			if (wbRequest == null) {
				wbRequest = new WaybackRequest();
			}
			
			logError(httpResponse, errorMsgHeader, e, wbRequest);
			
			LiveWebState liveWebState = LiveWebState.NOT_FOUND;
			
			if (getLiveWebRedirector() != null) {
				liveWebState = getLiveWebRedirector().handleRedirect(e, wbRequest, httpRequest, httpResponse);
			}
			
			// If not liveweb redirected, then render current exception
			if (liveWebState != LiveWebState.REDIRECTED) {
				e.setLiveWebAvailable(liveWebState == LiveWebState.FOUND);
				getException().renderException(httpRequest, httpResponse, wbRequest, e, getUriConverter());
			}
			
			handled = true;
			
		} catch (Exception other) {
			logError(httpResponse, errorMsgHeader, other, wbRequest);
		 } finally {      
	        //Slightly hacky, but ensures that all block loaders are closed
	        ZipNumBlockLoader.closeAllReaders();
	      }
		
		return handled;
	}
	
	public void logError(HttpServletResponse httpResponse, String header, Exception e, WaybackRequest request)
	{
		if (LOGGER.isLoggable(Level.INFO)) {
			if (e instanceof ResourceNotInArchiveException) {
				this.logNotInArchive((ResourceNotInArchiveException)e, request);
			} else {
				LOGGER.log(Level.INFO, "Runtime Error", e);
			}
		}
		
		if (!this.isEnableErrorMsgHeader()) {
			return;
		}
		
		String message = (e != null ? e.toString() : "");
		
		if (message == null) {
			message = "";
		} else {
			// Get substring from exception name
			int index = message.indexOf(':');
			if (index > 0) {
				index = message.lastIndexOf('.', index);
				if (index > 0) {
					message = message.substring(index + 1);
				}
			}
			
			if (message.length() > 200) {			
				message = message.substring(0, 200);
			}
			message = message.replace('\n', ' ');
		}
		
		httpResponse.setHeader(header, message);
	}
	
	private void logNotInArchive(ResourceNotInArchiveException e, WaybackRequest r) {
		// TODO: move this into ResourceNotInArchiveException constructor
		String url = r.getRequestUrl();
		StringBuilder sb = new StringBuilder(100);
		sb.append("NotInArchive\t");
		sb.append(getBeanName()).append("\t");
		sb.append(url);
		LOGGER.info(sb.toString());
	}

	protected void checkAccessPointAware(Object...os) {
		if(os != null) {
			for(Object o : os) {
				if(o instanceof AccessPointAware) {
					AccessPointAware apa = (AccessPointAware) o;
					apa.setAccessPoint(this);
				}
			}
		}
	}
	
	private void checkInterstitialRedirect(HttpServletRequest httpRequest,
			WaybackRequest wbRequest) 
	throws BetterRequestException {
		if((refererAuth != null) && (refererAuth.length() > 0)) {
			String referer = httpRequest.getHeader("Referer");
			if((referer != null) && (referer.length() > 0) && (!referer.contains(refererAuth))) {
				StringBuffer sb = httpRequest.getRequestURL();
				if(httpRequest.getQueryString() != null) {
					sb.append("?").append(httpRequest.getQueryString());
				}
				StringBuilder u = new StringBuilder();
				u.append(getQueryPrefix());
				u.append(interstitialJsp);
				u.append("?");
				u.append(INTERSTITIAL_SECONDS).append("=").append(5);
				u.append("&");
				u.append(INTERSTITIAL_DATE).append("=").append(wbRequest.getReplayDate().getTime());
				u.append("&");
				u.append(INTERSTITIAL_URL).append("=");
				try {
					u.append(URLEncoder.encode(wbRequest.getRequestUrl(), "UTF-8"));
				} catch (UnsupportedEncodingException e) {
					// not gonna happen...
					u.append(wbRequest.getRequestUrl());
				}
				u.append("&");
				u.append(INTERSTITIAL_TARGET).append("=");
				try {
					u.append(URLEncoder.encode(sb.toString(), "UTF-8"));
				} catch (UnsupportedEncodingException e) {
					// not gonna happen...
					u.append(sb.toString());
				}

				throw new BetterRequestException(u.toString());
			}
		}
	}
	
//	protected void addCustomHeaders(HttpServletResponse httpResponse, CaptureSearchResult closest)
//	{
//		Map<String, String> resultData = closest.toCanonicalStringMap();
//		
//		for (Entry<String, String> entry : resultData.entrySet()) {
//			String key = entry.getKey();
//			
//			if ((key != null) && key.startsWith(CaptureSearchResult.CUSTOM_HEADER_PREFIX)) {
//				key = key.substring(CaptureSearchResult.CUSTOM_HEADER_PREFIX.length());
//				String value = entry.getValue();
//				if (!key.isEmpty() && (value != null)) {
//					httpResponse.addHeader(key, value);
//				}
//			}
//		}
//	}
		
	protected boolean isSelfRedirect(Resource resource, CaptureSearchResult closest, WaybackRequest wbRequest, String canonRequestURL)
	{
		int status = resource.getStatusCode();
		
		// Only applies to redirects
		if ((status < 300) || (status >= 400)) {
			return false;
		}		
		
		String location = resource.getHeader("Location");
		
		if (location == null) {
			return false;
		}
					
//		if (!closest.getCaptureTimestamp().equals(wbRequest.getReplayTimestamp())) {
//			return false;			
//		}
		
		String redirScheme = UrlOperations.urlToScheme(location);
		
		try {	
			if (redirScheme == null && isExactSchemeMatch()) {
				location = UrlOperations.resolveUrl(closest.getOriginalUrl(), location);
				redirScheme = UrlOperations.urlToScheme(location);
			}
			
			if (getSelfRedirectCanonicalizer() != null) {
				location = getSelfRedirectCanonicalizer().urlStringToKey(location);
			}
		} catch (IOException e) {
			return false;
		}			
		
		if (location.equals(canonRequestURL)) {
			// if not exact scheme, don't do scheme compare, must be equal
			if (!isExactSchemeMatch()) {
				return true;
			}
			
			String origScheme = 
				UrlOperations.urlToScheme(wbRequest.getRequestUrl());
						
			if((origScheme != null) && (redirScheme != null) &&
					(origScheme.compareTo(redirScheme) == 0)) {
				return true;
			}
		}
		
		return false;
	}
	
	protected SearchResults queryIndex(WaybackRequest wbRequest) throws ResourceIndexNotAvailableException, ResourceNotInArchiveException, BadQueryException, AccessControlException, ConfigurationException
	{
		try {
			PerfStats.timeStart(PerfStat.IndexQueryTotal);
			return getCollection().getResourceIndex().query(wbRequest);
		} finally {
			PerfStats.timeEnd(PerfStat.IndexQueryTotal);			
		}
	}
	
	protected Resource getResource(CaptureSearchResult closest, Set<String> skipFiles) throws ResourceNotAvailableException, ConfigurationException
	{		
		try {
			PerfStats.timeStart(PerfStat.WArcResource);
			
			if ((skipFiles != null) && skipFiles.contains(closest.getFile())) {
				throw new ResourceNotAvailableException("Revisit: Skipping already failed " + closest.getFile());
			}
			
			return getCollection().getResourceStore().retrieveResource(closest);
		} finally {
			PerfStats.timeEnd(PerfStat.WArcResource);
		}
	}
	
	protected String getRedirectUrl(WaybackRequest wbRequest, String timestamp, String url)
	{
		String datespec = ArchivalUrl.getDateSpec(wbRequest, timestamp);
		String betterURI = getUriConverter().makeReplayURI(datespec, url);
		return betterURI;
	}
	
	protected void handleReplay(WaybackRequest wbRequest, 
			HttpServletRequest httpRequest, HttpServletResponse httpResponse) 
	throws IOException, ServletException, WaybackException {			
		
		checkInterstitialRedirect(httpRequest,wbRequest);
				
		String requestURL = wbRequest.getRequestUrl();
		
		if (getSelfRedirectCanonicalizer() != null) {
			try {
				requestURL = getSelfRedirectCanonicalizer().urlStringToKey(requestURL);
			} catch (IOException io) {
				
			}
		}
		
		long requestMS = Timestamp.parseBefore(wbRequest.getReplayTimestamp()).getDate().getTime();
		
		
		PerformanceLogger p = new PerformanceLogger("replay");
		
		// If optimized url+timestamp search is supported, mark the request
		if (this.isTimestampSearch()) {		
			if (wbRequest.isAnyEmbeddedContext() || wbRequest.isIdentityContext()) {
				wbRequest.setTimestampSearchKey(true);
			}
		}
		
		SearchResults results = queryIndex(wbRequest);
		p.queried();
		
		if(!(results instanceof CaptureSearchResults)) {
			throw new ResourceNotAvailableException("Bad results...");
		}
		CaptureSearchResults captureResults = 
			(CaptureSearchResults) results;

		
		CaptureSearchResult closest = null;
		
		closest = 
			getReplay().getClosest(wbRequest, captureResults);
		
		CaptureSearchResult originalClosest = closest;
		
		int counter = 0;
		
		//TODO: parameterize
		//int maxTimeouts = 2;
		//int maxMissingRevisits = 2;
		
		Set<String> skipFiles = null;
		//boolean isRevisit = false;
		
		while (true) {		
			// Support for redirect from the CDX redirectUrl field
			// This was the intended use of the redirect field, but has not actually be tested
			// To enable this functionality, uncomment the lines below
			// This is an optimization that allows for redirects to be handled without loading the original content
			//
			//String redir = closest.getRedirectUrl();
			//if ((redir != null) && !redir.equals("-")) {
			//  String fullRedirect = getUriConverter().makeReplayURI(closest.getCaptureTimestamp(), redir);
			//  throw new BetterRequestException(fullRedirect, Integer.valueOf(closest.getHttpCode()));
			//}
			
			Resource httpHeadersResource = null;
			Resource payloadResource = null;
			boolean isRevisit = false;
			
			try {
				counter++;
				
				if (closest == null) {
					throw new ResourceNotAvailableException("Self-Redirect: No Closest Match Found");
				}
				
				closest.setClosest(true);
				checkAnchorWindow(wbRequest,closest);
				
				// If revisit, may load two resources separately
				if (closest.isDuplicateDigest()) {
					isRevisit = true;
					
					// If the payload record is known and it failed before with this payload, don't try
					// loading the header resource even.. outcome will likely be same
					if ((closest.getDuplicatePayloadFile() != null) &&
						(skipFiles != null) && skipFiles.contains(closest.getDuplicatePayloadFile())) {
						counter--; //don't really count this as we're not even checking the file anymore
						throw new ResourceNotAvailableException("Revisit: Skipping already failed " + closest.getDuplicatePayloadFile());
					
					} else if ((closest.getDuplicatePayloadFile() == null) && wbRequest.isTimestampSearchKey()) {
						// If a missing revisit and loaded optimized, try loading the entire timeline again
						
						wbRequest.setTimestampSearchKey(false);
						
						results = queryIndex(wbRequest);
						
						captureResults = (CaptureSearchResults)results;
						
						closest = getReplay().getClosest(wbRequest, captureResults);
						originalClosest = closest;
						//maxTimeouts *= 2;
						//maxMissingRevisits *= 2;
						
						continue;
					}
					
					// If old-style arc revisit (no mimetype, filename is '-'), then don't load
					// headersResource = payloadResource
					if (EMPTY_VALUE.equals(closest.getFile())) {
						closest.setFile(closest.getDuplicatePayloadFile());
						closest.setOffset(closest.getDuplicatePayloadOffset());
						
						// See that this is successful
						httpHeadersResource = getResource(closest, skipFiles);
						
						// Then, if both headers and payload are from a different timestamp, redirect to that timestamp
						if (!closest.getCaptureTimestamp().equals(closest.getDuplicateDigestStoredTimestamp())) {
							throw new BetterRequestException(
									getRedirectUrl(wbRequest, closest.getDuplicateDigestStoredTimestamp(), closest.getOriginalUrl()));
						}
						
						payloadResource = httpHeadersResource;
						
					} else {
						httpHeadersResource = getResource(closest, skipFiles);
						payloadResource = retrievePayloadForIdenticalContentRevisit(httpHeadersResource, captureResults, closest, skipFiles);
						
						// If zero length old-style revisit with no headers, then must use payloadResource as headersResource
						if (httpHeadersResource.getRecordLength() <= 0) {
							httpHeadersResource = payloadResource;
						}
					}
				} else {
					httpHeadersResource = getResource(closest, skipFiles);
					payloadResource = httpHeadersResource;
				}
				
				// Ensure that we are not self-redirecting!
				// If the status is a redirect, check that the location or url date's are different from the current request
				// Otherwise, replay the previous matched capture.
				// This chain is unlikely to go past one previous capture, but is possible 
				if (isSelfRedirect(httpHeadersResource, closest, wbRequest, requestURL)) {
					LOGGER.info("Self-Redirect: Skipping " + closest.getCaptureTimestamp() + "/" + closest.getOriginalUrl());
					closest = findNextClosest(closest, captureResults, requestMS);
					continue;
				}
				
				// Attempt to resolve any not-found embedded content with closest
				if (wbRequest.isAnyEmbeddedContext() && closest.getHttpCode().startsWith("4")) {
					CaptureSearchResult nextClosest = findNextClosest(closest, captureResults, requestMS);
					if ((nextClosest != null) && !nextClosest.getHttpCode().startsWith("4")) {
						closest = nextClosest;
						continue;<commit_msg>FIX: Close old httpHeadersResource before setting httpHeadersResource = payloadResource<commit_after>/*
 *  This file is part of the Wayback archival access software
 *   (http://archive-access.sourceforge.net/projects/wayback/).
 *
 *  Licensed to the Internet Archive (IA) by one or more individual 
 *  contributors. 
 *
 *  The IA licenses this file to You under the Apache License, Version 2.0
 *  (the "License"); you may not use this file except in compliance with
 *  the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
package org.archive.wayback.webapp;

import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.archive.format.gzip.zipnum.ZipNumBlockLoader;
import org.archive.wayback.ExceptionRenderer;
import org.archive.wayback.QueryRenderer;
import org.archive.wayback.ReplayDispatcher;
import org.archive.wayback.ReplayRenderer;
import org.archive.wayback.RequestParser;
import org.archive.wayback.ResultURIConverter;
import org.archive.wayback.UrlCanonicalizer;
import org.archive.wayback.accesscontrol.ExclusionFilterFactory;
import org.archive.wayback.archivalurl.ArchivalUrl;
import org.archive.wayback.core.CaptureSearchResult;
import org.archive.wayback.core.CaptureSearchResults;
import org.archive.wayback.core.Resource;
import org.archive.wayback.core.SearchResults;
import org.archive.wayback.core.UIResults;
import org.archive.wayback.core.UrlSearchResults;
import org.archive.wayback.core.WaybackRequest;
import org.archive.wayback.exception.AccessControlException;
import org.archive.wayback.exception.AdministrativeAccessControlException;
import org.archive.wayback.exception.AnchorWindowTooSmallException;
import org.archive.wayback.exception.AuthenticationControlException;
import org.archive.wayback.exception.BadQueryException;
import org.archive.wayback.exception.BaseExceptionRenderer;
import org.archive.wayback.exception.BetterRequestException;
import org.archive.wayback.exception.ConfigurationException;
import org.archive.wayback.exception.ResourceIndexNotAvailableException;
import org.archive.wayback.exception.ResourceNotAvailableException;
import org.archive.wayback.exception.ResourceNotInArchiveException;
import org.archive.wayback.exception.SpecificCaptureReplayException;
import org.archive.wayback.exception.WaybackException;
import org.archive.wayback.resourceindex.filters.ExclusionFilter;
import org.archive.wayback.resourceindex.filters.WARCRevisitAnnotationFilter;
import org.archive.wayback.resourcestore.resourcefile.WarcResource;
import org.archive.wayback.util.Timestamp;
import org.archive.wayback.util.operator.BooleanOperator;
import org.archive.wayback.util.url.UrlOperations;
import org.archive.wayback.util.webapp.AbstractRequestHandler;
import org.archive.wayback.util.webapp.ShutdownListener;
import org.archive.wayback.webapp.LiveWebRedirector.LiveWebState;

/**
 * Retains all information about a particular Wayback configuration
 * within a ServletContext, including holding references to the
 * implementation instances of the primary Wayback classes:
 * 
 * 		RequestParser
 *		ResourceIndex(via WaybackCollection)
 *		ResourceStore(via WaybackCollection)
 *		QueryRenderer
 *		ReplayDispatcher
 *		ExceptionRenderer
 *		ResultURIConverter
 *
 *
 * @author brad
 * @version $Date$, $Revision$
 */
public class AccessPoint extends AbstractRequestHandler 
implements ShutdownListener {
	/** webapp relative location of Interstitial.jsp */
	public final static String INTERSTITIAL_JSP = "jsp/Interstitial.jsp";
	/** argument for Interstitial.jsp target URL */
	public final static String INTERSTITIAL_TARGET = "target";
	/** argument for Interstitial.jsp seconds to delay */
	public final static String INTERSTITIAL_SECONDS = "seconds";

	/** argument for Interstitial.jsp msse for replay date */
	public final static String INTERSTITIAL_DATE = "date";
	/** argument for Interstitial.jsp URL being loaded */
	public final static String INTERSTITIAL_URL = "url";
	
	public final static String REVISIT_STR = "warc/revisit";
	public final static String EMPTY_VALUE = "-";
	
	public final static String RUNTIME_ERROR_HEADER = "X-Archive-Wayback-Runtime-Error";
	//public final static String NOTFOUND_ERROR_HEADER = "X-Archive-Wayback-Not-Found";

	private static final Logger LOGGER = Logger.getLogger(
			AccessPoint.class.getName());

	private boolean exactHostMatch = false;
	private boolean exactSchemeMatch = false;
	private boolean useAnchorWindow = false;
	private boolean useServerName = false;
	private boolean serveStatic = true;
	private boolean bounceToReplayPrefix = false;
	private boolean bounceToQueryPrefix = false;
	private boolean forceCleanQueries = true;

	private boolean timestampSearch = false;
	
	public static enum PerfStat
	{
		IndexQueryTotal,
		WArcResource,
		Total,
	}
	
	private String errorMsgHeader = RUNTIME_ERROR_HEADER;
	private String perfStatsHeader = "X-Archive-Wayback-Perf";
	private String warcFileHeader = "x-archive-src";
	
	private boolean enableErrorMsgHeader = false;
	private boolean enablePerfStatsHeader = false;
	private boolean enableWarcFileHeader = false;
		
	private LiveWebRedirector liveWebRedirector;
	
	private String staticPrefix = null;
	private String queryPrefix = null;
	private String replayPrefix = null;
	
	private String interstitialJsp = INTERSTITIAL_JSP;

	private String refererAuth = null;

	private Locale locale = null;

	private Properties configs = null;

	private List<String> filePatterns = null;
	private List<String> fileIncludePrefixes = null;
	private List<String> fileExcludePrefixes = null;

	private WaybackCollection  collection   = null;
	private ExceptionRenderer  exception    = new BaseExceptionRenderer();
	private QueryRenderer      query        = null;
	private RequestParser      parser       = null;
	private ReplayDispatcher   replay       = null;
	private ResultURIConverter uriConverter = null;

	private ExclusionFilterFactory exclusionFactory = null;
	private BooleanOperator<WaybackRequest> authentication = null;
	private boolean requestAuth = true;
	
	private long embargoMS = 0;
	private CustomResultFilterFactory filterFactory = null;
	
	private UrlCanonicalizer selfRedirectCanonicalizer = null;
	
	private int maxRedirectAttempts = 0;

	public void init() {
		checkAccessPointAware(collection,exception,query,parser,replay,
				uriConverter,exclusionFactory, authentication, filterFactory);
	}
	
	protected boolean dispatchLocal(HttpServletRequest httpRequest,
			HttpServletResponse httpResponse) 
	throws ServletException, IOException {
		if(LOGGER.isLoggable(Level.FINE)) {
			LOGGER.fine("Local dispatch /" + translateRequestPath(httpRequest));
		}
		if(!serveStatic) {
			return false;
		}
//		String contextRelativePath = httpRequest.getServletPath();
		String translatedNoQuery = "/" + translateRequestPath(httpRequest);
//		String absPath = getServletContext().getRealPath(contextRelativePath);
		String absPath = getServletContext().getRealPath(translatedNoQuery);
		
		//IK: added null check for absPath, it may be null (ex. on jetty)
		if (absPath != null) {
			File test = new File(absPath);
			if((test != null) && !test.exists()) {
				return false;
			}
		}
				
		String translatedQ = "/" + translateRequestPathQuery(httpRequest);

		WaybackRequest wbRequest = new WaybackRequest();
//			wbRequest.setContextPrefix(getUrlRoot());
		wbRequest.setAccessPoint(this);
		wbRequest.fixup(httpRequest);
		UIResults uiResults = new UIResults(wbRequest,uriConverter);
		try {
			uiResults.forward(httpRequest, httpResponse, translatedQ);
			return true;
		} catch(IOException e) {
			// TODO: figure out if we got IO because of a missing dispatcher
		}

		return false;
	}

	/**
	 * @param httpRequest HttpServletRequest which is being handled 
	 * @param httpResponse HttpServletResponse which is being handled 
	 * @return true if the request was actually handled
	 * @throws ServletException per usual
	 * @throws IOException per usual
	 */
	public boolean handleRequest(HttpServletRequest httpRequest,
			HttpServletResponse httpResponse) 
	throws ServletException, IOException {

		WaybackRequest wbRequest = null;
		boolean handled = false;

		try {
			PerfStats.clearAll();
			
			if (this.isEnablePerfStatsHeader() && (perfStatsHeader != null)) {
				PerfStats.timeStart(PerfStat.Total);
				httpResponse = new PerfWritingHttpServletResponse(httpResponse, PerfStat.Total, perfStatsHeader);
			}
			
			String inputPath = translateRequestPathQuery(httpRequest);
			Thread.currentThread().setName("Thread " + 
					Thread.currentThread().getId() + " " + getBeanName() + 
					" handling: " + inputPath);
			LOGGER.fine("Handling translated: " + inputPath);
			wbRequest = getParser().parse(httpRequest, this);

			if(wbRequest != null) {
				handled = true;

				// TODO: refactor this code into RequestParser implementations
				wbRequest.setAccessPoint(this);
//				wbRequest.setContextPrefix(getAbsoluteLocalPrefix(httpRequest));
//				wbRequest.setContextPrefix(getUrlRoot());
				wbRequest.fixup(httpRequest);
				// end of refactor

				if(getAuthentication() != null) {
					if(!getAuthentication().isTrue(wbRequest)) {
						throw new AuthenticationControlException(
								"Unauthorized", isRequestAuth());
					}
				}

				if(getExclusionFactory() != null) {
					ExclusionFilter exclusionFilter = 
						getExclusionFactory().get();
					if(exclusionFilter == null) {
						throw new AdministrativeAccessControlException(
								"AccessControl list unavailable");
					}
					wbRequest.setExclusionFilter(exclusionFilter);
				}
				// TODO: refactor this into RequestParser implementations, so a
				// user could alter requests to change the behavior within a
				// single AccessPoint. For now, this is a simple way to expose
				// the feature to configuration.g
				wbRequest.setExactScheme(isExactSchemeMatch());

				if(wbRequest.isReplayRequest()) {
					if(bounceToReplayPrefix) {
						// we don't accept replay requests on this AccessPoint
						// bounce the user to the right place:
						String suffix = translateRequestPathQuery(httpRequest);
						String replayUrl = replayPrefix + suffix;
						httpResponse.sendRedirect(replayUrl);
						return true;
					}
					handleReplay(wbRequest,httpRequest,httpResponse);
					
				} else {

					if(bounceToQueryPrefix) {
						// we don't accept replay requests on this AccessPoint
						// bounce the user to the right place:
						String suffix = translateRequestPathQuery(httpRequest);
						String replayUrl = queryPrefix + suffix;
						httpResponse.sendRedirect(replayUrl);
						return true;
					}
					wbRequest.setExactHost(isExactHostMatch());
					handleQuery(wbRequest,httpRequest,httpResponse);
				}
			} else {
				handled = dispatchLocal(httpRequest,httpResponse);
			}
			
		} catch(BetterRequestException e) {
			e.generateResponse(httpResponse);
			handled = true;

		} catch(WaybackException e) {
			
			if (wbRequest == null) {
				wbRequest = new WaybackRequest();
			}
			
			logError(httpResponse, errorMsgHeader, e, wbRequest);
			
			LiveWebState liveWebState = LiveWebState.NOT_FOUND;
			
			if (getLiveWebRedirector() != null) {
				liveWebState = getLiveWebRedirector().handleRedirect(e, wbRequest, httpRequest, httpResponse);
			}
			
			// If not liveweb redirected, then render current exception
			if (liveWebState != LiveWebState.REDIRECTED) {
				e.setLiveWebAvailable(liveWebState == LiveWebState.FOUND);
				getException().renderException(httpRequest, httpResponse, wbRequest, e, getUriConverter());
			}
			
			handled = true;
			
		} catch (Exception other) {
			logError(httpResponse, errorMsgHeader, other, wbRequest);
		 } finally {      
	        //Slightly hacky, but ensures that all block loaders are closed
	        ZipNumBlockLoader.closeAllReaders();
	      }
		
		return handled;
	}
	
	public void logError(HttpServletResponse httpResponse, String header, Exception e, WaybackRequest request)
	{
		if (LOGGER.isLoggable(Level.INFO)) {
			if (e instanceof ResourceNotInArchiveException) {
				this.logNotInArchive((ResourceNotInArchiveException)e, request);
			} else {
				LOGGER.log(Level.INFO, "Runtime Error", e);
			}
		}
		
		if (!this.isEnableErrorMsgHeader()) {
			return;
		}
		
		String message = (e != null ? e.toString() : "");
		
		if (message == null) {
			message = "";
		} else {
			// Get substring from exception name
			int index = message.indexOf(':');
			if (index > 0) {
				index = message.lastIndexOf('.', index);
				if (index > 0) {
					message = message.substring(index + 1);
				}
			}
			
			if (message.length() > 200) {			
				message = message.substring(0, 200);
			}
			message = message.replace('\n', ' ');
		}
		
		httpResponse.setHeader(header, message);
	}
	
	private void logNotInArchive(ResourceNotInArchiveException e, WaybackRequest r) {
		// TODO: move this into ResourceNotInArchiveException constructor
		String url = r.getRequestUrl();
		StringBuilder sb = new StringBuilder(100);
		sb.append("NotInArchive\t");
		sb.append(getBeanName()).append("\t");
		sb.append(url);
		LOGGER.info(sb.toString());
	}

	protected void checkAccessPointAware(Object...os) {
		if(os != null) {
			for(Object o : os) {
				if(o instanceof AccessPointAware) {
					AccessPointAware apa = (AccessPointAware) o;
					apa.setAccessPoint(this);
				}
			}
		}
	}
	
	private void checkInterstitialRedirect(HttpServletRequest httpRequest,
			WaybackRequest wbRequest) 
	throws BetterRequestException {
		if((refererAuth != null) && (refererAuth.length() > 0)) {
			String referer = httpRequest.getHeader("Referer");
			if((referer != null) && (referer.length() > 0) && (!referer.contains(refererAuth))) {
				StringBuffer sb = httpRequest.getRequestURL();
				if(httpRequest.getQueryString() != null) {
					sb.append("?").append(httpRequest.getQueryString());
				}
				StringBuilder u = new StringBuilder();
				u.append(getQueryPrefix());
				u.append(interstitialJsp);
				u.append("?");
				u.append(INTERSTITIAL_SECONDS).append("=").append(5);
				u.append("&");
				u.append(INTERSTITIAL_DATE).append("=").append(wbRequest.getReplayDate().getTime());
				u.append("&");
				u.append(INTERSTITIAL_URL).append("=");
				try {
					u.append(URLEncoder.encode(wbRequest.getRequestUrl(), "UTF-8"));
				} catch (UnsupportedEncodingException e) {
					// not gonna happen...
					u.append(wbRequest.getRequestUrl());
				}
				u.append("&");
				u.append(INTERSTITIAL_TARGET).append("=");
				try {
					u.append(URLEncoder.encode(sb.toString(), "UTF-8"));
				} catch (UnsupportedEncodingException e) {
					// not gonna happen...
					u.append(sb.toString());
				}

				throw new BetterRequestException(u.toString());
			}
		}
	}
	
//	protected void addCustomHeaders(HttpServletResponse httpResponse, CaptureSearchResult closest)
//	{
//		Map<String, String> resultData = closest.toCanonicalStringMap();
//		
//		for (Entry<String, String> entry : resultData.entrySet()) {
//			String key = entry.getKey();
//			
//			if ((key != null) && key.startsWith(CaptureSearchResult.CUSTOM_HEADER_PREFIX)) {
//				key = key.substring(CaptureSearchResult.CUSTOM_HEADER_PREFIX.length());
//				String value = entry.getValue();
//				if (!key.isEmpty() && (value != null)) {
//					httpResponse.addHeader(key, value);
//				}
//			}
//		}
//	}
		
	protected boolean isSelfRedirect(Resource resource, CaptureSearchResult closest, WaybackRequest wbRequest, String canonRequestURL)
	{
		int status = resource.getStatusCode();
		
		// Only applies to redirects
		if ((status < 300) || (status >= 400)) {
			return false;
		}		
		
		String location = resource.getHeader("Location");
		
		if (location == null) {
			return false;
		}
					
//		if (!closest.getCaptureTimestamp().equals(wbRequest.getReplayTimestamp())) {
//			return false;			
//		}
		
		String redirScheme = UrlOperations.urlToScheme(location);
		
		try {	
			if (redirScheme == null && isExactSchemeMatch()) {
				location = UrlOperations.resolveUrl(closest.getOriginalUrl(), location);
				redirScheme = UrlOperations.urlToScheme(location);
			}
			
			if (getSelfRedirectCanonicalizer() != null) {
				location = getSelfRedirectCanonicalizer().urlStringToKey(location);
			}
		} catch (IOException e) {
			return false;
		}			
		
		if (location.equals(canonRequestURL)) {
			// if not exact scheme, don't do scheme compare, must be equal
			if (!isExactSchemeMatch()) {
				return true;
			}
			
			String origScheme = 
				UrlOperations.urlToScheme(wbRequest.getRequestUrl());
						
			if((origScheme != null) && (redirScheme != null) &&
					(origScheme.compareTo(redirScheme) == 0)) {
				return true;
			}
		}
		
		return false;
	}
	
	protected SearchResults queryIndex(WaybackRequest wbRequest) throws ResourceIndexNotAvailableException, ResourceNotInArchiveException, BadQueryException, AccessControlException, ConfigurationException
	{
		try {
			PerfStats.timeStart(PerfStat.IndexQueryTotal);
			return getCollection().getResourceIndex().query(wbRequest);
		} finally {
			PerfStats.timeEnd(PerfStat.IndexQueryTotal);			
		}
	}
	
	protected Resource getResource(CaptureSearchResult closest, Set<String> skipFiles) throws ResourceNotAvailableException, ConfigurationException
	{		
		try {
			PerfStats.timeStart(PerfStat.WArcResource);
			
			if ((skipFiles != null) && skipFiles.contains(closest.getFile())) {
				throw new ResourceNotAvailableException("Revisit: Skipping already failed " + closest.getFile());
			}
			
			return getCollection().getResourceStore().retrieveResource(closest);
		} finally {
			PerfStats.timeEnd(PerfStat.WArcResource);
		}
	}
	
	protected String getRedirectUrl(WaybackRequest wbRequest, String timestamp, String url)
	{
		String datespec = ArchivalUrl.getDateSpec(wbRequest, timestamp);
		String betterURI = getUriConverter().makeReplayURI(datespec, url);
		return betterURI;
	}
	
	protected void handleReplay(WaybackRequest wbRequest, 
			HttpServletRequest httpRequest, HttpServletResponse httpResponse) 
	throws IOException, ServletException, WaybackException {			
		
		checkInterstitialRedirect(httpRequest,wbRequest);
				
		String requestURL = wbRequest.getRequestUrl();
		
		if (getSelfRedirectCanonicalizer() != null) {
			try {
				requestURL = getSelfRedirectCanonicalizer().urlStringToKey(requestURL);
			} catch (IOException io) {
				
			}
		}
		
		long requestMS = Timestamp.parseBefore(wbRequest.getReplayTimestamp()).getDate().getTime();
		
		
		PerformanceLogger p = new PerformanceLogger("replay");
		
		// If optimized url+timestamp search is supported, mark the request
		if (this.isTimestampSearch()) {		
			if (wbRequest.isAnyEmbeddedContext() || wbRequest.isIdentityContext()) {
				wbRequest.setTimestampSearchKey(true);
			}
		}
		
		SearchResults results = queryIndex(wbRequest);
		p.queried();
		
		if(!(results instanceof CaptureSearchResults)) {
			throw new ResourceNotAvailableException("Bad results...");
		}
		CaptureSearchResults captureResults = 
			(CaptureSearchResults) results;

		
		CaptureSearchResult closest = null;
		
		closest = 
			getReplay().getClosest(wbRequest, captureResults);
		
		CaptureSearchResult originalClosest = closest;
		
		int counter = 0;
		
		//TODO: parameterize
		//int maxTimeouts = 2;
		//int maxMissingRevisits = 2;
		
		Set<String> skipFiles = null;
		//boolean isRevisit = false;
		
		while (true) {		
			// Support for redirect from the CDX redirectUrl field
			// This was the intended use of the redirect field, but has not actually be tested
			// To enable this functionality, uncomment the lines below
			// This is an optimization that allows for redirects to be handled without loading the original content
			//
			//String redir = closest.getRedirectUrl();
			//if ((redir != null) && !redir.equals("-")) {
			//  String fullRedirect = getUriConverter().makeReplayURI(closest.getCaptureTimestamp(), redir);
			//  throw new BetterRequestException(fullRedirect, Integer.valueOf(closest.getHttpCode()));
			//}
			
			Resource httpHeadersResource = null;
			Resource payloadResource = null;
			boolean isRevisit = false;
			
			try {
				counter++;
				
				if (closest == null) {
					throw new ResourceNotAvailableException("Self-Redirect: No Closest Match Found");
				}
				
				closest.setClosest(true);
				checkAnchorWindow(wbRequest,closest);
				
				// If revisit, may load two resources separately
				if (closest.isDuplicateDigest()) {
					isRevisit = true;
					
					// If the payload record is known and it failed before with this payload, don't try
					// loading the header resource even.. outcome will likely be same
					if ((closest.getDuplicatePayloadFile() != null) &&
						(skipFiles != null) && skipFiles.contains(closest.getDuplicatePayloadFile())) {
						counter--; //don't really count this as we're not even checking the file anymore
						throw new ResourceNotAvailableException("Revisit: Skipping already failed " + closest.getDuplicatePayloadFile());
					
					} else if ((closest.getDuplicatePayloadFile() == null) && wbRequest.isTimestampSearchKey()) {
						// If a missing revisit and loaded optimized, try loading the entire timeline again
						
						wbRequest.setTimestampSearchKey(false);
						
						results = queryIndex(wbRequest);
						
						captureResults = (CaptureSearchResults)results;
						
						closest = getReplay().getClosest(wbRequest, captureResults);
						originalClosest = closest;
						//maxTimeouts *= 2;
						//maxMissingRevisits *= 2;
						
						continue;
					}
					
					// If old-style arc revisit (no mimetype, filename is '-'), then don't load
					// headersResource = payloadResource
					if (EMPTY_VALUE.equals(closest.getFile())) {
						closest.setFile(closest.getDuplicatePayloadFile());
						closest.setOffset(closest.getDuplicatePayloadOffset());
						
						// See that this is successful
						httpHeadersResource = getResource(closest, skipFiles);
						
						// Then, if both headers and payload are from a different timestamp, redirect to that timestamp
						if (!closest.getCaptureTimestamp().equals(closest.getDuplicateDigestStoredTimestamp())) {
							throw new BetterRequestException(
									getRedirectUrl(wbRequest, closest.getDuplicateDigestStoredTimestamp(), closest.getOriginalUrl()));
						}
						
						payloadResource = httpHeadersResource;
						
					} else {
						httpHeadersResource = getResource(closest, skipFiles);
						payloadResource = retrievePayloadForIdenticalContentRevisit(httpHeadersResource, captureResults, closest, skipFiles);
						
						// If zero length old-style revisit with no headers, then must use payloadResource as headersResource
						if (httpHeadersResource.getRecordLength() <= 0) {
							httpHeadersResource.close();
							httpHeadersResource = payloadResource;
						}
					}
				} else {
					httpHeadersResource = getResource(closest, skipFiles);
					payloadResource = httpHeadersResource;
				}
				
				// Ensure that we are not self-redirecting!
				// If the status is a redirect, check that the location or url date's are different from the current request
				// Otherwise, replay the previous matched capture.
				// This chain is unlikely to go past one previous capture, but is possible 
				if (isSelfRedirect(httpHeadersResource, closest, wbRequest, requestURL)) {
					LOGGER.info("Self-Redirect: Skipping " + closest.getCaptureTimestamp() + "/" + closest.getOriginalUrl());
					closest = findNextClosest(closest, captureResults, requestMS);
					continue;
				}
				
				// Attempt to resolve any not-found embedded content with closest
				if (wbRequest.isAnyEmbeddedContext() && closest.getHttpCode().startsWith("4")) {
					CaptureSearchResult nextClosest = findNextClosest(closest, captureResults, requestMS);
					if ((nextClosest != null) && !nextClosest.getHttpCode().startsWith("4")) {
						closest = nextClosest;
						continue;
{'fraction_non_alphanumeric': 0.10177423292911207, 'fraction_numerical': 0.0015870432164075852, 'mean_word_length': 4.037822878228782, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6817636', 'n_tokens_mistral': 16579, 'n_tokens_neox': 13697, 'n_words': 3351}