<commit_before>package org.archive.wayback.resourceindex.cdxserver;

import java.util.HashMap;
import java.util.LinkedList;

import org.apache.commons.lang.math.NumberUtils;
import org.archive.cdxserver.CDXQuery;
import org.archive.format.cdx.CDXLine;
import org.archive.wayback.core.CaptureSearchResult;
import org.archive.wayback.core.CaptureSearchResults;
import org.archive.wayback.core.FastCaptureSearchResult;
import org.archive.wayback.resourceindex.filters.SelfRedirectFilter;
import org.archive.wayback.util.ObjectFilter;
import org.archive.wayback.util.Timestamp;
import org.archive.wayback.util.url.UrlOperations;

public class CDXToCaptureSearchResultsWriter extends CDXToSearchResultWriter {
	
	public final static String REVISIT_VALUE = "warc/revisit";
	
	protected CaptureSearchResults results = null;
	
	protected String targetTimestamp;
	protected int flip = 1;
	protected boolean done = false;
	protected CaptureSearchResult closest = null;
	protected SelfRedirectFilter selfRedirFilter = null;
	
	protected HashMap<String, CaptureSearchResult> digestToOriginal;
	protected HashMap<String, LinkedList<CaptureSearchResult>> digestToRevisits;
	
	protected boolean resolveRevisits = false;
	protected boolean seekSingleCapture = false;
	protected boolean isReverse = false;
	
	public CDXToCaptureSearchResultsWriter(CDXQuery query,
										   boolean resolveRevisits, 
										   boolean seekSingleCapture)
	{
		super(query);
		
		this.resolveRevisits = resolveRevisits;
		this.seekSingleCapture = seekSingleCapture;
		this.isReverse = query.isReverse();
	}
	
	public void setTargetTimestamp(String timestamp)
	{		
		targetTimestamp = timestamp;
		
		if (isReverse) {
			flip = -1;
		}
	}

	@Override
    public void begin() {
		results = new CaptureSearchResults();
		
		if (resolveRevisits) {
			if (isReverse) {
				digestToRevisits = new HashMap<String, LinkedList<CaptureSearchResult>>();
			} else {
				digestToOriginal = new HashMap<String, CaptureSearchResult>();
			}
		}
    }

	@Override
    public int writeLine(CDXLine line) {
		FastCaptureSearchResult result = new FastCaptureSearchResult();
		
		result.setUrlKey(line.getUrlKey());
		result.setCaptureTimestamp(line.getTimestamp());
		result.setOriginalUrl(line.getOriginalUrl());
		
		// Special case: filter out captures that have userinfo
		boolean hasUserInfo = (UrlOperations.urlToUserInfo(result.getOriginalUrl()) != null);
		
		if (hasUserInfo) {
			return 0;
		}
		
		result.setRedirectUrl(line.getRedirect());
		result.setHttpCode(line.getStatusCode());
		
		if (selfRedirFilter != null && !result.getRedirectUrl().equals(CDXLine.EMPTY_VALUE)) {
			if (selfRedirFilter.filterObject(result) != ObjectFilter.FILTER_INCLUDE) {
				return 0;
			}
		}
		
		result.setMimeType(line.getMimeType());
		result.setDigest(line.getDigest());
		result.setOffset(NumberUtils.toLong(line.getOffset(), -1));
		result.setCompressedLength(NumberUtils.toLong(line.getLength(), -1));
		result.setFile(line.getFilename());
		result.setRobotFlags(line.getRobotFlags());
		
		boolean isRevisit = false;
		
		if (resolveRevisits) {
			isRevisit = result.getFile().equals(CDXLine.EMPTY_VALUE) ||
						result.getMimeType().equals(REVISIT_VALUE);
			
			String digest = result.getDigest();
			
			if (isRevisit) {
				if (!isReverse) {
					CaptureSearchResult payload = digestToOriginal.get(digest);
					if (payload != null) {
						result.flagDuplicateDigest(payload);
					}
				} else {
					LinkedList<CaptureSearchResult> revisits = digestToRevisits.get(digest);
					if (revisits == null) {
						revisits = new LinkedList<CaptureSearchResult>();
						digestToRevisits.put(digest, revisits);
					}
					revisits.add(result);
				}
			} else {
				if (!isReverse) {
					digestToOriginal.put(digest, result);
				} else {
					LinkedList<CaptureSearchResult> revisits = digestToRevisits.remove(digest);
					if (revisits != null) {
						for (CaptureSearchResult revisit : revisits) {
							revisit.flagDuplicateDigest(result);
						}
					}
				}
			}
		}
		
//		String payloadFile = line.getField(RevisitResolver.origfilename);
//		
//		if (!payloadFile.equals(CDXLine.EMPTY_VALUE)) {
//			FastCaptureSearchResult payload = new FastCaptureSearchResult();
//			payload.setFile(payloadFile);
//			payload.setOffset(NumberUtils.toLong(line.getField(RevisitResolver.origoffset), -1));
//			payload.setCompressedLength(NumberUtils.toLong(line.getField(RevisitResolver.origlength), -1));
//			result.flagDuplicateDigest(payload);
//		}
		
		if ((targetTimestamp != null) && (closest == null)) {
			closest = determineClosest(result);
		}
		
		results.addSearchResult(result, !isReverse);
		
		// Short circuit the load if seeking single capture
		if (seekSingleCapture && resolveRevisits) {
			if (closest != null) {
				// If not a revisit, we're done
				if (!isRevisit) {<commit_msg>FIX: skip exact duplicates of same url and timestamp<commit_after>package org.archive.wayback.resourceindex.cdxserver;

import java.util.HashMap;
import java.util.LinkedList;

import org.apache.commons.lang.math.NumberUtils;
import org.archive.cdxserver.CDXQuery;
import org.archive.format.cdx.CDXLine;
import org.archive.wayback.core.CaptureSearchResult;
import org.archive.wayback.core.CaptureSearchResults;
import org.archive.wayback.core.FastCaptureSearchResult;
import org.archive.wayback.resourceindex.filters.SelfRedirectFilter;
import org.archive.wayback.util.ObjectFilter;
import org.archive.wayback.util.Timestamp;
import org.archive.wayback.util.url.UrlOperations;

public class CDXToCaptureSearchResultsWriter extends CDXToSearchResultWriter {
	
	public final static String REVISIT_VALUE = "warc/revisit";
	
	protected CaptureSearchResults results = null;
	
	protected String targetTimestamp;
	protected int flip = 1;
	protected boolean done = false;
	protected CaptureSearchResult closest = null;
	protected SelfRedirectFilter selfRedirFilter = null;
	
	protected CaptureSearchResult lastResult = null;
	
	protected HashMap<String, CaptureSearchResult> digestToOriginal;
	protected HashMap<String, LinkedList<CaptureSearchResult>> digestToRevisits;
	
	protected boolean resolveRevisits = false;
	protected boolean seekSingleCapture = false;
	protected boolean isReverse = false;
	
	public CDXToCaptureSearchResultsWriter(CDXQuery query,
										   boolean resolveRevisits, 
										   boolean seekSingleCapture)
	{
		super(query);
		
		this.resolveRevisits = resolveRevisits;
		this.seekSingleCapture = seekSingleCapture;
		this.isReverse = query.isReverse();
	}
	
	public void setTargetTimestamp(String timestamp)
	{		
		targetTimestamp = timestamp;
		
		if (isReverse) {
			flip = -1;
		}
	}

	@Override
    public void begin() {
		results = new CaptureSearchResults();
		
		if (resolveRevisits) {
			if (isReverse) {
				digestToRevisits = new HashMap<String, LinkedList<CaptureSearchResult>>();
			} else {
				digestToOriginal = new HashMap<String, CaptureSearchResult>();
			}
		}
    }

	@Override
    public int writeLine(CDXLine line) {
		FastCaptureSearchResult result = new FastCaptureSearchResult();
		
		String timestamp = line.getTimestamp();
		
		if (lastResult != null) {
			if (lastResult.getCaptureTimestamp().equals(timestamp)) {
				// Skip this
				return 0;
			}
		}
				
		result.setUrlKey(line.getUrlKey());
		result.setCaptureTimestamp(timestamp);
		result.setOriginalUrl(line.getOriginalUrl());
		
		// Special case: filter out captures that have userinfo
		boolean hasUserInfo = (UrlOperations.urlToUserInfo(result.getOriginalUrl()) != null);
		
		if (hasUserInfo) {
			return 0;
		}
		
		result.setRedirectUrl(line.getRedirect());
		result.setHttpCode(line.getStatusCode());
		
		if (selfRedirFilter != null && !result.getRedirectUrl().equals(CDXLine.EMPTY_VALUE)) {
			if (selfRedirFilter.filterObject(result) != ObjectFilter.FILTER_INCLUDE) {
				return 0;
			}
		}
		
		result.setMimeType(line.getMimeType());
		result.setDigest(line.getDigest());
		result.setOffset(NumberUtils.toLong(line.getOffset(), -1));
		result.setCompressedLength(NumberUtils.toLong(line.getLength(), -1));
		result.setFile(line.getFilename());
		result.setRobotFlags(line.getRobotFlags());
		
		boolean isRevisit = false;
		
		if (resolveRevisits) {
			isRevisit = result.getFile().equals(CDXLine.EMPTY_VALUE) ||
						result.getMimeType().equals(REVISIT_VALUE);
			
			String digest = result.getDigest();
			
			if (isRevisit) {
				if (!isReverse) {
					CaptureSearchResult payload = digestToOriginal.get(digest);
					if (payload != null) {
						result.flagDuplicateDigest(payload);
					}
				} else {
					LinkedList<CaptureSearchResult> revisits = digestToRevisits.get(digest);
					if (revisits == null) {
						revisits = new LinkedList<CaptureSearchResult>();
						digestToRevisits.put(digest, revisits);
					}
					revisits.add(result);
				}
			} else {
				if (!isReverse) {
					digestToOriginal.put(digest, result);
				} else {
					LinkedList<CaptureSearchResult> revisits = digestToRevisits.remove(digest);
					if (revisits != null) {
						for (CaptureSearchResult revisit : revisits) {
							revisit.flagDuplicateDigest(result);
						}
					}
				}
			}
		}
		
//		String payloadFile = line.getField(RevisitResolver.origfilename);
//		
//		if (!payloadFile.equals(CDXLine.EMPTY_VALUE)) {
//			FastCaptureSearchResult payload = new FastCaptureSearchResult();
//			payload.setFile(payloadFile);
//			payload.setOffset(NumberUtils.toLong(line.getField(RevisitResolver.origoffset), -1));
//			payload.setCompressedLength(NumberUtils.toLong(line.getField(RevisitResolver.origlength), -1));
//			result.flagDuplicateDigest(payload);
//		}
		
		if ((targetTimestamp != null) && (closest == null)) {
			closest = determineClosest(result);
		}
		
		results.addSearchResult(result, !isReverse);
		lastResult = result;
		
		// Short circuit the load if seeking single capture
		if (seekSingleCapture && resolveRevisits) {
			if (closest != null) {
				// If not a revisit, we're done
				if (!isRevisit) {
{'fraction_non_alphanumeric': 0.11114442778610695, 'fraction_numerical': 0.001699150424787606, 'mean_word_length': 4.2196139801773604, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2376795', 'n_tokens_mistral': 3649, 'n_tokens_neox': 3038, 'n_words': 564}