<commit_before>package com.commafeed.backend.services;

import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;

import javax.inject.Inject;

import lombok.extern.slf4j.Slf4j;

import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;

/**
 * Contains utility methods for cleaning the database
 * 
 */
@Slf4j
public class DatabaseCleaningService {

	private static final int BATCH_SIZE = 100;

	@Inject
	FeedDAO feedDAO;

	@Inject
	FeedEntryDAO feedEntryDAO;

	@Inject
	FeedSubscriptionDAO feedSubscriptionDAO;

	@Inject
	FeedEntryContentDAO feedEntryContentDAO;

	@Inject
	FeedEntryStatusDAO feedEntryStatusDAO;

	@Inject
	ApplicationSettingsService applicationSettingsService;

	public long cleanFeedsWithoutSubscriptions() {
		log.info("cleaning feeds without subscriptions");
		long total = 0;
		long totalEntries = 0;
		int deleted = 0;
		do {
			List<Feed> feeds = feedDAO.findWithoutSubscriptions(1);
			totalEntries += cleanEntriesForFeedsWithoutSubscriptions(feeds);
			deleted = feedDAO.delete(feeds);
			total += deleted;
			log.info("removed {} feeds without subscriptions ({} entries)", total, totalEntries);
		} while (deleted != 0);
		log.info("cleanup done: {} feeds without subscriptions deleted", total);
		return total;
	}

	private long cleanEntriesForFeedsWithoutSubscriptions(List<Feed> feeds) {
		long total = 0;
		for (Feed feed : feeds) {
			int deleted = 0;
			do {
				deleted = feedEntryDAO.delete(feed, BATCH_SIZE);
				total += deleted;
			} while (deleted != 0);
		}
		return total;
	}

	public long cleanContentsWithoutEntries() {
		log.info("cleaning contents without entries");
		long total = 0;
		int deleted = 0;
		do {<commit_msg>more logging<commit_after>package com.commafeed.backend.services;

import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;

import javax.inject.Inject;

import lombok.extern.slf4j.Slf4j;

import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;

/**
 * Contains utility methods for cleaning the database
 * 
 */
@Slf4j
public class DatabaseCleaningService {

	private static final int BATCH_SIZE = 100;

	@Inject
	FeedDAO feedDAO;

	@Inject
	FeedEntryDAO feedEntryDAO;

	@Inject
	FeedSubscriptionDAO feedSubscriptionDAO;

	@Inject
	FeedEntryContentDAO feedEntryContentDAO;

	@Inject
	FeedEntryStatusDAO feedEntryStatusDAO;

	@Inject
	ApplicationSettingsService applicationSettingsService;

	public long cleanFeedsWithoutSubscriptions() {
		log.info("cleaning feeds without subscriptions");
		long total = 0;
		int deleted = 0;
		do {
			List<Feed> feeds = feedDAO.findWithoutSubscriptions(1);
			cleanEntriesForFeedsWithoutSubscriptions(feeds);
			deleted = feedDAO.delete(feeds);
			total += deleted;
			log.info("removed {} feeds without subscriptions", total);
		} while (deleted != 0);
		log.info("cleanup done: {} feeds without subscriptions deleted", total);
		return total;
	}

	private long cleanEntriesForFeedsWithoutSubscriptions(List<Feed> feeds) {
		long total = 0;
		for (Feed feed : feeds) {
			int deleted = 0;
			do {
				deleted = feedEntryDAO.delete(feed, BATCH_SIZE);
				total += deleted;
				log.info("removed {} entries for feed {}", total, feed.getId());
			} while (deleted != 0);
		}
		return total;
	}

	public long cleanContentsWithoutEntries() {
		log.info("cleaning contents without entries");
		long total = 0;
		int deleted = 0;
		do {
{'fraction_non_alphanumeric': 0.10047846889952153, 'fraction_numerical': 0.007416267942583732, 'mean_word_length': 4.964336661911555, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2145633', 'n_tokens_mistral': 1442, 'n_tokens_neox': 1349, 'n_words': 303}