<commit_before>package com.clxcommunications.xms;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import org.apache.http.concurrent.FutureCallback;

import com.clxcommunications.xms.api.Page;

/**
 * Used for API calls that emits their result over multiple pages. This class
 * includes convenience methods for interacting with such pages in various ways.
 * For example, it is possible to retrieve individual pages or to produce an
 * iterator that seamlessly will iterate over all elements of all pages.
 * 
 * @param <T>
 *            the element type
 */
public abstract class PagedFetcher<T> {

	/**
	 * Synchronously fetches the page having the given page number.
	 * 
	 * @param page
	 *            the page number (starting from zero)
	 * @return the fetched page
	 * @throws ConcurrentException
	 *             if another checked exception occurred during execution
	 * @throws ErrorResponseException
	 *             if the HTTP endpoint returned an error message
	 * @throws UnexpectedResponseException
	 *             if the HTTP endpoint responded with an unexpected message
	 * @throws InterruptedException
	 *             if the fetch was interrupted before completing
	 */
	@Nonnull
	Page<T> fetch(int page) throws InterruptedException, ConcurrentException,
	        ErrorResponseException, UnexpectedResponseException {
		try {
			return fetchAsync(page, null).get();
		} catch (ExecutionException e) {
			throw Utils.unwrapExecutionException(e);
		}
	}

	/**
	 * Asynchronously fetches the page having the given page number.
	 * 
	 * @param page
	 *            page to fetch (staring from zero)
	 * @param callback
	 *            request callback
	 * @return a future providing the requested page
	 */
	@Nonnull
	abstract Future<Page<T>> fetchAsync(int page,
	        @Nullable FutureCallback<Page<T>> callback);

	/**
	 * Fetch all pages, some or all pages may be fetched asynchronously.
	 * <p>
	 * The returned list is ordered by page number but no guarantee is made of
	 * the order in which the callback is called.
	 * 
	 * @param callback
	 *            request callback, called once per page
	 * @return a queue of futures, one for each fetched page
	 */
	@Nonnull
	List<Future<Page<T>>> fetchPagesAsync(
	        @Nullable final FutureCallback<Page<T>> callback) {
		final Queue<Future<Page<T>>> futures =
		        new ConcurrentLinkedQueue<Future<Page<T>>>();

		Future<Page<T>> initialFuture =
		        fetchAsync(0, new FutureCallback<Page<T>>() {

			        @Override
			        public void completed(Page<T> result) {
				        callback.completed(result);

				        /*
				         * Go over all remaining pages and fetch them. Note,
				         * they are all submitted to the HTTP client library
				         * asynchronously!
				         */
				        for (int i = 1; i < result.numPages(); i++) {
					        futures.add(fetchAsync(i, callback));
				        }
			        }

			        @Override
			        public void failed(Exception ex) {
				        callback.failed(ex);
			        }

			        @Override
			        public void cancelled() {
				        callback.cancelled();
			        }

		        });

		List<Future<Page<T>>> result =
		        new ArrayList<Future<Page<T>>>(1 + futures.size());

		result.add(initialFuture);
		result.addAll(futures);

		return result;
	}

	/**
	 * Returns an iterable object that traverses all fetched elements across all
	 * associated pages. This is done by iterating over fetched pages and, when
	 * necessary, fetching new pages.<commit_msg>Remove "all pages at the same time" fetcher method<commit_after>package com.clxcommunications.xms;

import java.util.Iterator;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import org.apache.http.concurrent.FutureCallback;

import com.clxcommunications.xms.api.Page;

/**
 * Used for API calls that emits their result over multiple pages. This class
 * includes convenience methods for interacting with such pages in various ways.
 * For example, it is possible to retrieve individual pages or to produce an
 * iterator that seamlessly will iterate over all elements of all pages.
 * 
 * @param <T>
 *            the element type
 */
public abstract class PagedFetcher<T> {

	/**
	 * Synchronously fetches the page having the given page number.
	 * 
	 * @param page
	 *            the page number (starting from zero)
	 * @return the fetched page
	 * @throws ConcurrentException
	 *             if another checked exception occurred during execution
	 * @throws ErrorResponseException
	 *             if the HTTP endpoint returned an error message
	 * @throws UnexpectedResponseException
	 *             if the HTTP endpoint responded with an unexpected message
	 * @throws InterruptedException
	 *             if the fetch was interrupted before completing
	 */
	@Nonnull
	Page<T> fetch(int page) throws InterruptedException, ConcurrentException,
	        ErrorResponseException, UnexpectedResponseException {
		try {
			return fetchAsync(page, null).get();
		} catch (ExecutionException e) {
			throw Utils.unwrapExecutionException(e);
		}
	}

	/**
	 * Asynchronously fetches the page having the given page number.
	 * 
	 * @param page
	 *            page to fetch (staring from zero)
	 * @param callback
	 *            request callback
	 * @return a future providing the requested page
	 */
	@Nonnull
	abstract Future<Page<T>> fetchAsync(int page,
	        @Nullable FutureCallback<Page<T>> callback);

	/**
	 * Returns an iterable object that traverses all fetched elements across all
	 * associated pages. This is done by iterating over fetched pages and, when
	 * necessary, fetching new pages.
{'fraction_non_alphanumeric': 0.07932011331444759, 'fraction_numerical': 0.0004999166805532411, 'mean_word_length': 2.9331585845347314, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '825059', 'n_tokens_mistral': 1692, 'n_tokens_neox': 1492, 'n_words': 566}