<commit_before>package com.vaguehope.onosendai.update;

import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;

import twitter4j.TwitterException;

import com.vaguehope.onosendai.config.Account;
import com.vaguehope.onosendai.config.Column;
import com.vaguehope.onosendai.model.Tweet;
import com.vaguehope.onosendai.model.TweetList;
import com.vaguehope.onosendai.provider.ProviderMgr;
import com.vaguehope.onosendai.provider.successwhale.SuccessWhaleException;
import com.vaguehope.onosendai.provider.successwhale.SuccessWhaleFeed;
import com.vaguehope.onosendai.provider.successwhale.SuccessWhaleProvider;
import com.vaguehope.onosendai.provider.twitter.TwitterFeed;
import com.vaguehope.onosendai.provider.twitter.TwitterFeeds;
import com.vaguehope.onosendai.provider.twitter.TwitterProvider;
import com.vaguehope.onosendai.provider.twitter.TwitterUtils;
import com.vaguehope.onosendai.storage.DbInterface;
import com.vaguehope.onosendai.storage.DbInterface.ColumnState;
import com.vaguehope.onosendai.util.LogWrapper;

public class FetchColumn implements Callable<Void> {

	protected static final LogWrapper LOG = new LogWrapper("FC");

	private final DbInterface db;
	private final Account account;
	private final Column column;
	private final ProviderMgr providerMgr;

	public FetchColumn (final DbInterface db, final Account account, final Column column, final ProviderMgr providerMgr) {
		if (db == null) throw new IllegalArgumentException("db can not be null.");
		if (account == null) throw new IllegalArgumentException("account can not be null.");
		if (column == null) throw new IllegalArgumentException("column can not be null.");
		if (providerMgr == null) throw new IllegalArgumentException("providerMgr can not be null.");
		this.db = db;
		this.account = account;
		this.column = column;
		this.providerMgr = providerMgr;
	}

	@Override
	public Void call () {
		fetchColumn(this.db, this.account, this.column, this.providerMgr);
		return null;
	}

	public static void fetchColumn (final DbInterface db, final Account account, final Column column, final ProviderMgr providerMgr) {
		db.notifyTwListenersColumnState(column.getId(), ColumnState.UPDATE_RUNNING);
		try {
			fetchColumnInner(db, account, column, providerMgr);
		}
		finally {
			db.notifyTwListenersColumnState(column.getId(), ColumnState.UPDATE_OVER);
		}
	}

	private static void fetchColumnInner (final DbInterface db, final Account account, final Column column, final ProviderMgr providerMgr) {
		final long startTime = System.nanoTime();
		switch (account.getProvider()) {
			case TWITTER:
				try {
					final TwitterProvider twitterProvider = providerMgr.getTwitterProvider();
					twitterProvider.addAccount(account);
					TwitterFeed feed = TwitterFeeds.parse(column.getResource());

					long sinceId = -1;
					List<Tweet> existingTweets = db.getTweets(column.getId(), 1);
					if (existingTweets.size() > 0) sinceId = Long.parseLong(existingTweets.get(existingTweets.size() - 1).getSid());

					TweetList tweets = twitterProvider.getTweets(feed, account, sinceId);
					db.storeTweets(column, tweets.getTweets());

					storeSuccess(db, column);
					long durationMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
					LOG.i("Fetched %d items for '%s' in %d millis.", tweets.count(), column.getTitle(), durationMillis);
				}
				catch (TwitterException e) {
					LOG.w("Failed to fetch from Twitter: %s", e.toString());
					storeError(db, column, TwitterUtils.friendlyExceptionMessage(e));
				}
				break;
			case SUCCESSWHALE:
				try {
					final SuccessWhaleProvider successWhaleProvider = providerMgr.getSuccessWhaleProvider();
					successWhaleProvider.addAccount(account);
					SuccessWhaleFeed feed = new SuccessWhaleFeed(column);

					String sinceId = null;
					List<Tweet> existingTweets = db.getTweets(column.getId(), 1);
					if (existingTweets.size() > 0) sinceId = existingTweets.get(existingTweets.size() - 1).getSid();

					TweetList tweets = successWhaleProvider.getTweets(feed, account, sinceId);
					db.storeTweets(column, tweets.getTweets());
<commit_msg>Do not bother telling DB to store nothing.<commit_after>package com.vaguehope.onosendai.update;

import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;

import twitter4j.TwitterException;

import com.vaguehope.onosendai.config.Account;
import com.vaguehope.onosendai.config.Column;
import com.vaguehope.onosendai.model.Tweet;
import com.vaguehope.onosendai.model.TweetList;
import com.vaguehope.onosendai.provider.ProviderMgr;
import com.vaguehope.onosendai.provider.successwhale.SuccessWhaleException;
import com.vaguehope.onosendai.provider.successwhale.SuccessWhaleFeed;
import com.vaguehope.onosendai.provider.successwhale.SuccessWhaleProvider;
import com.vaguehope.onosendai.provider.twitter.TwitterFeed;
import com.vaguehope.onosendai.provider.twitter.TwitterFeeds;
import com.vaguehope.onosendai.provider.twitter.TwitterProvider;
import com.vaguehope.onosendai.provider.twitter.TwitterUtils;
import com.vaguehope.onosendai.storage.DbInterface;
import com.vaguehope.onosendai.storage.DbInterface.ColumnState;
import com.vaguehope.onosendai.util.LogWrapper;

public class FetchColumn implements Callable<Void> {

	protected static final LogWrapper LOG = new LogWrapper("FC");

	private final DbInterface db;
	private final Account account;
	private final Column column;
	private final ProviderMgr providerMgr;

	public FetchColumn (final DbInterface db, final Account account, final Column column, final ProviderMgr providerMgr) {
		if (db == null) throw new IllegalArgumentException("db can not be null.");
		if (account == null) throw new IllegalArgumentException("account can not be null.");
		if (column == null) throw new IllegalArgumentException("column can not be null.");
		if (providerMgr == null) throw new IllegalArgumentException("providerMgr can not be null.");
		this.db = db;
		this.account = account;
		this.column = column;
		this.providerMgr = providerMgr;
	}

	@Override
	public Void call () {
		fetchColumn(this.db, this.account, this.column, this.providerMgr);
		return null;
	}

	public static void fetchColumn (final DbInterface db, final Account account, final Column column, final ProviderMgr providerMgr) {
		db.notifyTwListenersColumnState(column.getId(), ColumnState.UPDATE_RUNNING);
		try {
			fetchColumnInner(db, account, column, providerMgr);
		}
		finally {
			db.notifyTwListenersColumnState(column.getId(), ColumnState.UPDATE_OVER);
		}
	}

	private static void fetchColumnInner (final DbInterface db, final Account account, final Column column, final ProviderMgr providerMgr) {
		final long startTime = System.nanoTime();
		switch (account.getProvider()) {
			case TWITTER:
				try {
					final TwitterProvider twitterProvider = providerMgr.getTwitterProvider();
					twitterProvider.addAccount(account);
					TwitterFeed feed = TwitterFeeds.parse(column.getResource());

					long sinceId = -1;
					List<Tweet> existingTweets = db.getTweets(column.getId(), 1);
					if (existingTweets.size() > 0) sinceId = Long.parseLong(existingTweets.get(existingTweets.size() - 1).getSid());

					TweetList tweets = twitterProvider.getTweets(feed, account, sinceId);
					if (tweets.count() > 0) db.storeTweets(column, tweets.getTweets());

					storeSuccess(db, column);
					long durationMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
					LOG.i("Fetched %d items for '%s' in %d millis.", tweets.count(), column.getTitle(), durationMillis);
				}
				catch (TwitterException e) {
					LOG.w("Failed to fetch from Twitter: %s", e.toString());
					storeError(db, column, TwitterUtils.friendlyExceptionMessage(e));
				}
				break;
			case SUCCESSWHALE:
				try {
					final SuccessWhaleProvider successWhaleProvider = providerMgr.getSuccessWhaleProvider();
					successWhaleProvider.addAccount(account);
					SuccessWhaleFeed feed = new SuccessWhaleFeed(column);

					String sinceId = null;
					List<Tweet> existingTweets = db.getTweets(column.getId(), 1);
					if (existingTweets.size() > 0) sinceId = existingTweets.get(existingTweets.size() - 1).getSid();

					TweetList tweets = successWhaleProvider.getTweets(feed, account, sinceId);
					if (tweets.count() > 0) db.storeTweets(column, tweets.getTweets());

{'fraction_non_alphanumeric': 0.10547108823177302, 'fraction_numerical': 0.0021549144020112532, 'mean_word_length': 5.138133725202057, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5934965', 'n_tokens_mistral': 2878, 'n_tokens_neox': 2553, 'n_words': 577}