<commit_before>/*
 * The MIT License
 *
 * Copyright 2014-2015 TweetWallFX
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
package org.tweetwallfx.tweet.impl.twitter4j;

import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.apache.log4j.Logger;
import org.tweetwallfx.tweet.api.Tweet;
import org.tweetwallfx.tweet.api.TweetStream;
import org.tweetwallfx.tweet.api.Tweeter;
import org.tweetwallfx.tweet.api.TweetQuery;
import twitter4j.Query;
import twitter4j.QueryResult;
import twitter4j.Status;
import twitter4j.Twitter;
import twitter4j.TwitterException;
import twitter4j.TwitterFactory;

public class TwitterTweeter extends Tweeter {

    private static final Logger LOGGER = Logger.getLogger(TwitterTweeter.class);

    public TwitterTweeter() {
        TwitterOAuth.exception().addListener((observable, oldValue, newValue) -> setLatestException(newValue));
        TwitterOAuth.getConfiguration();
    }

    @Override
    public TweetStream createTweetStream() {
        return new TwitterTweetStream();
    }

    @Override
    public Stream<Tweet> search(final TweetQuery tweetQuery) {
        final Twitter twitter = new TwitterFactory(TwitterOAuth.getConfiguration()).getInstance();
        final Query query = getQuery(tweetQuery);
        final QueryResult result;

        try {
            result = twitter.search(query);
        } catch (TwitterException ex) {
            setLatestException(ex);
            LOGGER.error("Error getting QueryResult for " + query, ex);
            return Stream.empty();
        }

        return result.getTweets().stream().map(TwitterTweet::new);
    }

    @Override
    public Stream<Tweet> searchPaged(final TweetQuery tweetQuery) {
        final Query query = getQuery(tweetQuery);
        final Iterable<Tweet> iterable = () -> new PagedIterator(this, query);
        return StreamSupport.stream(iterable.spliterator(), false);
    }

    private static Query getQuery(final TweetQuery tweetQuery) {
        final Query query = new Query();
        
        if (null != tweetQuery.getCount()) {
            query.setCount(tweetQuery.getCount());
        }
        
        if (null != tweetQuery.getLang()) {
            query.setLang(tweetQuery.getLang());
        }
        
        if (null != tweetQuery.getLocale()) {
            query.setLocale(tweetQuery.getLocale());
        }
        
        if (null != tweetQuery.getMaxId()) {
            query.setMaxId(tweetQuery.getMaxId());
        }
        
        if (null != tweetQuery.getQuery()) {
            query.setQuery(tweetQuery.getQuery());
        }
        
        if (null != tweetQuery.getResultType()) {
            query.setResultType(Query.ResultType.valueOf(tweetQuery.getResultType().name()));
        }
        
        if (null != tweetQuery.getSince()) {
            query.setSince(tweetQuery.getSince());
        }
        
        if (null != tweetQuery.getSinceId()) {
            query.setSinceId(tweetQuery.getSinceId());
        }
        
        if (null != tweetQuery.getUntil()) {
            query.setUntil(tweetQuery.getUntil());
        }
        
        return query;
//        return new Query()
//                .count(tweetQuery.getCount())
//                .lang(tweetQuery.getLang())
//                .locale(tweetQuery.getLocale())
//                .maxId(tweetQuery.getMaxId())
//                .query(tweetQuery.getQuery())
//                .resultType(null == tweetQuery.getResultType()
//                        ? null
//                        : Query.ResultType.valueOf(tweetQuery.getResultType().name()))
//                .since(tweetQuery.getSince())
//                .sinceId(tweetQuery.getSinceId())
//                .until(tweetQuery.getUntil());
    }

    private static class PagedIterator implements Iterator<Tweet> {

        private final TwitterTweeter tweeter;
        private QueryResult queryResult;
        private Iterator<Status> statuses;

        public PagedIterator(final TwitterTweeter tweeter, final Query query) {
            this.tweeter = tweeter;
            queryNext(query);
        }

        private void queryNext(final Query query) {
            if (null == query) {
                statuses = null;
            } else {
                final Twitter twitter = new TwitterFactory(TwitterOAuth.getConfiguration()).getInstance();

                try {
                    queryResult = twitter.search(query);
                    statuses = queryResult.getTweets().iterator();
                } catch (TwitterException ex) {
                    tweeter.setLatestException(ex);
                    LOGGER.error("Error getting QueryResult for " + query, ex);
                    queryResult = null;
                    statuses = null;
                }
            }
        }

        @Override
        public boolean hasNext() {
            if (null == statuses) {
                // no more twitter status messages
                return false;<commit_msg>removed unused lines<commit_after>/*
 * The MIT License
 *
 * Copyright 2014-2015 TweetWallFX
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
package org.tweetwallfx.tweet.impl.twitter4j;

import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.apache.log4j.Logger;
import org.tweetwallfx.tweet.api.Tweet;
import org.tweetwallfx.tweet.api.TweetStream;
import org.tweetwallfx.tweet.api.Tweeter;
import org.tweetwallfx.tweet.api.TweetQuery;
import twitter4j.Query;
import twitter4j.QueryResult;
import twitter4j.Status;
import twitter4j.Twitter;
import twitter4j.TwitterException;
import twitter4j.TwitterFactory;

public class TwitterTweeter extends Tweeter {

    private static final Logger LOGGER = Logger.getLogger(TwitterTweeter.class);

    public TwitterTweeter() {
        TwitterOAuth.exception().addListener((observable, oldValue, newValue) -> setLatestException(newValue));
        TwitterOAuth.getConfiguration();
    }

    @Override
    public TweetStream createTweetStream() {
        return new TwitterTweetStream();
    }

    @Override
    public Stream<Tweet> search(final TweetQuery tweetQuery) {
        final Twitter twitter = new TwitterFactory(TwitterOAuth.getConfiguration()).getInstance();
        final Query query = getQuery(tweetQuery);
        final QueryResult result;

        try {
            result = twitter.search(query);
        } catch (TwitterException ex) {
            setLatestException(ex);
            LOGGER.error("Error getting QueryResult for " + query, ex);
            return Stream.empty();
        }

        return result.getTweets().stream().map(TwitterTweet::new);
    }

    @Override
    public Stream<Tweet> searchPaged(final TweetQuery tweetQuery) {
        final Query query = getQuery(tweetQuery);
        final Iterable<Tweet> iterable = () -> new PagedIterator(this, query);
        return StreamSupport.stream(iterable.spliterator(), false);
    }

    private static Query getQuery(final TweetQuery tweetQuery) {
        final Query query = new Query();

        if (null != tweetQuery.getCount()) {
            query.setCount(tweetQuery.getCount());
        }

        if (null != tweetQuery.getLang()) {
            query.setLang(tweetQuery.getLang());
        }

        if (null != tweetQuery.getLocale()) {
            query.setLocale(tweetQuery.getLocale());
        }

        if (null != tweetQuery.getMaxId()) {
            query.setMaxId(tweetQuery.getMaxId());
        }

        if (null != tweetQuery.getQuery()) {
            query.setQuery(tweetQuery.getQuery());
        }

        if (null != tweetQuery.getResultType()) {
            query.setResultType(Query.ResultType.valueOf(tweetQuery.getResultType().name()));
        }

        if (null != tweetQuery.getSince()) {
            query.setSince(tweetQuery.getSince());
        }

        if (null != tweetQuery.getSinceId()) {
            query.setSinceId(tweetQuery.getSinceId());
        }

        if (null != tweetQuery.getUntil()) {
            query.setUntil(tweetQuery.getUntil());
        }

        return query;
    }

    private static class PagedIterator implements Iterator<Tweet> {

        private final TwitterTweeter tweeter;
        private QueryResult queryResult;
        private Iterator<Status> statuses;
        private static final Logger startupLogger = Logger.getLogger("org.tweetwallfx.startup");

        public PagedIterator(final TwitterTweeter tweeter, final Query query) {
            this.tweeter = tweeter;
            queryNext(query);
        }

        private void queryNext(final Query query) {
            if (null == query) {
                statuses = null;
            } else {
                final Twitter twitter = new TwitterFactory(TwitterOAuth.getConfiguration()).getInstance();

                try {
                    startupLogger.trace("Querying next page: " + query);
                    queryResult = twitter.search(query);
                    statuses = queryResult.getTweets().iterator();
                } catch (TwitterException ex) {
                    startupLogger.trace("Querying next page failed: " + query, ex);
                    tweeter.setLatestException(ex);
                    LOGGER.error("Error getting QueryResult for " + query, ex);
                    queryResult = null;
                    statuses = null;
                }
            }
        }

        @Override
        public boolean hasNext() {
            if (null == statuses) {
                // no more twitter status messages
                return false;
{'fraction_non_alphanumeric': 0.0945531914893617, 'fraction_numerical': 0.002723404255319149, 'mean_word_length': 2.40905134899913, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4049529', 'n_tokens_mistral': 3191, 'n_tokens_neox': 3012, 'n_words': 840}