<commit_before>package ikube.action.index.handler.internet;

import ikube.action.index.handler.IResourceProvider;
import ikube.model.IndexableInternet;
import ikube.model.Url;
import ikube.toolkit.HashUtilities;
import ikube.toolkit.ThreadUtilities;

import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.Stack;
import java.util.TreeSet;

class InternetResourceProvider implements IResourceProvider<Url> {

	private Stack<Url> urls;
	private Set<Long> done;

	InternetResourceProvider(final IndexableInternet indexableInternet) {
		urls = new Stack<Url>();
		done = new TreeSet<Long>();
		Url url = new Url();
		url.setUrl(indexableInternet.getUrl());
		url.setName(indexableInternet.getName());
		setResources(Arrays.asList(url));
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public Url getResource() {
		if (urls.isEmpty()) {
			// We'll wait a few seconds to see if any other thread will add some urls to the stack
			ThreadUtilities.sleep(600000);
			if (urls.isEmpty()) {
				return null;
			} else {<commit_msg>Reduced the wait time for the internet handler threads for resources.<commit_after>package ikube.action.index.handler.internet;

import ikube.action.index.handler.IResourceProvider;
import ikube.model.IndexableInternet;
import ikube.model.Url;
import ikube.toolkit.HashUtilities;
import ikube.toolkit.ThreadUtilities;

import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.Stack;
import java.util.TreeSet;

class InternetResourceProvider implements IResourceProvider<Url> {

	private Stack<Url> urls;
	private Set<Long> done;

	InternetResourceProvider(final IndexableInternet indexableInternet) {
		urls = new Stack<Url>();
		done = new TreeSet<Long>();
		Url url = new Url();
		url.setUrl(indexableInternet.getUrl());
		url.setName(indexableInternet.getName());
		setResources(Arrays.asList(url));
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public Url getResource() {
		if (urls.isEmpty()) {
			// We'll wait a few seconds to see if any other thread will add some urls to the stack
			ThreadUtilities.sleep(10000);
			if (urls.isEmpty()) {
				return null;
			} else {
{'fraction_non_alphanumeric': 0.11565257779842081, 'fraction_numerical': 0.005109150023223409, 'mean_word_length': 4.853260869565218, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6444537', 'n_tokens_mistral': 727, 'n_tokens_neox': 674, 'n_words': 168}