<commit_before>package stsc.news.feedzilla.downloader;

import graef.feedzillajava.Article;
import graef.feedzillajava.Articles;
import graef.feedzillajava.Category;
import graef.feedzillajava.FeedZilla;
import graef.feedzillajava.Subcategory;

import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
import java.util.TimeZone;
import java.util.concurrent.Callable;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.config.XMLConfigurationFactory;

class CallableArticlesDownload implements Callable<Optional<List<Article>>> {

	static {
		System.setProperty(XMLConfigurationFactory.CONFIGURATION_FILE_PROPERTY, "./config/log4j2.xml");
		TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
	}

	private static Logger callableLogger = LogManager.getLogger(CallableArticlesDownload.class);

	public static final int TRIES_COUNT = 7;
	public static final long PAUSE_SLEEP_TIME = 230;

	private final FeedZilla feed;
	private final Category category;
	private final Subcategory subcategory;
	private final int amountOfArticlesPerRequest;
	private final LocalDateTime startOfDay;

	public CallableArticlesDownload(FeedZilla feed, Category category, Subcategory subcategory, int amountOfArticlesPerRequest,
			LocalDateTime startOfDay) {
		super();
		this.feed = feed;
		this.category = category;
		this.subcategory = subcategory;
		this.amountOfArticlesPerRequest = amountOfArticlesPerRequest;
		this.startOfDay = startOfDay;
	}

	@Override
	public Optional<List<Article>> call() throws Exception {
		final long startArticlesLoadTime = System.currentTimeMillis();
		Exception exceptionToReturn = new Exception();
		for (int amountOfTries = 0; amountOfTries < TRIES_COUNT; ++amountOfTries) {
			try {
				final Articles articles = feed.query().category(category.getId()).subcategory(subcategory.getId()).since(startOfDay)
						.count(amountOfArticlesPerRequest).articles();
				final Optional<List<Article>> result = Optional.of(articles.getArticles());
				final long endArticlesLoadTime = System.currentTimeMillis();
				callableLogger.trace("articles load took: " + (endArticlesLoadTime - startArticlesLoadTime) + " ms " + (result.get().size()));
				return result;
			} catch (Exception e) {
				exceptionToReturn = e;
			}
			pause();
		}
		final long endArticlesLoadTime = System.currentTimeMillis();
		callableLogger.trace("no articles and it took: " + (endArticlesLoadTime - startArticlesLoadTime) + " ms; "
				+ exceptionToReturn.getMessage());
		return Optional.empty();
	}

	public static void pause() {
		pause(PAUSE_SLEEP_TIME);
	}

	public static void pause(long timeInMillis) {
		try {
			Thread.sleep(timeInMillis);
		} catch (Exception e) {
		}
	}

}
<commit_msg>#44 added category, subcategory names to CallableArticlesDownload trace log files<commit_after>package stsc.news.feedzilla.downloader;

import graef.feedzillajava.Article;
import graef.feedzillajava.Articles;
import graef.feedzillajava.Category;
import graef.feedzillajava.FeedZilla;
import graef.feedzillajava.Subcategory;

import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
import java.util.TimeZone;
import java.util.concurrent.Callable;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.config.XMLConfigurationFactory;

class CallableArticlesDownload implements Callable<Optional<List<Article>>> {

	static {
		System.setProperty(XMLConfigurationFactory.CONFIGURATION_FILE_PROPERTY, "./config/log4j2.xml");
		TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
	}

	private static Logger callableLogger = LogManager.getLogger(CallableArticlesDownload.class);

	public static final int TRIES_COUNT = 7;
	public static final long PAUSE_SLEEP_TIME = 230;

	private final FeedZilla feed;
	private final Category category;
	private final Subcategory subcategory;
	private final int amountOfArticlesPerRequest;
	private final LocalDateTime startOfDay;

	public CallableArticlesDownload(FeedZilla feed, Category category, Subcategory subcategory, int amountOfArticlesPerRequest,
			LocalDateTime startOfDay) {
		super();
		this.feed = feed;
		this.category = category;
		this.subcategory = subcategory;
		this.amountOfArticlesPerRequest = amountOfArticlesPerRequest;
		this.startOfDay = startOfDay;
	}

	@Override
	public Optional<List<Article>> call() throws Exception {
		final long startArticlesLoadTime = System.currentTimeMillis();
		Exception exceptionToReturn = new Exception();
		for (int amountOfTries = 0; amountOfTries < TRIES_COUNT; ++amountOfTries) {
			try {
				final Articles articles = feed.query().category(category.getId()).subcategory(subcategory.getId()).since(startOfDay)
						.count(amountOfArticlesPerRequest).articles();
				final Optional<List<Article>> result = Optional.of(articles.getArticles());
				final long endArticlesLoadTime = System.currentTimeMillis();
				callableLogger.trace("articles load took: " + (endArticlesLoadTime - startArticlesLoadTime) + " ms "
						+ (result.get().size()) + " Category(" + category.getDisplayName() + "), Subcategory("
						+ subcategory.getDisplayName() + ")");
				return result;
			} catch (Exception e) {
				exceptionToReturn = e;
			}
			pause();
		}
		final long endArticlesLoadTime = System.currentTimeMillis();
		callableLogger.trace("no articles and it took: " + (endArticlesLoadTime - startArticlesLoadTime) + " ms 0 Category("
				+ category.getDisplayName() + "), Subcategory(" + subcategory.getDisplayName() + ")" + exceptionToReturn.getMessage());
		return Optional.empty();
	}

	public static void pause() {
		pause(PAUSE_SLEEP_TIME);
	}

	public static void pause(long timeInMillis) {
		try {
			Thread.sleep(timeInMillis);
		} catch (Exception e) {
		}
	}

}

{'fraction_non_alphanumeric': 0.10414160508678467, 'fraction_numerical': 0.003952569169960474, 'mean_word_length': 5.481069042316259, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7327267', 'n_tokens_mistral': 1902, 'n_tokens_neox': 1747, 'n_words': 368}