<commit_before>from time import sleep

import pandas as pd
import tweepy

from constants import MAX_TWEET_LENGTH, TWEET_THRESHOLD, TIME_BETWEEN_TWEETS
from credentials import *
from predict import add_predictions_to_date
from preprocessing import load_arxiv_and_tweets


def get_title_tweet(date):
    return 'arXiv papers published on {}:'.format(date.strftime('%B %-m, %Y'))


def tweet_latest_day(dry_run=True):
    df = load_arxiv_and_tweets()
    index_to_predict_for = get_published_on_day_index(df)
    df = add_predictions_to_date(df, index_to_predict_for)
    predicted_papers = df.loc[index_to_predict_for]
    to_tweet = predicted_papers[predicted_papers.prediction > TWEET_THRESHOLD]
    if not to_tweet.empty:
        published_on = to_tweet.iloc[0].published.date()
        paper_tweets = to_tweet.sort_values('prediction', ascending=False).apply(get_tweet_text, axis=1)
        title_tweet = get_title_tweet(published_on)
        to_tweet = [title_tweet] + list(paper_tweets.values)

        if dry_run:
            for t in to_tweet:
                print t
                print

        elif published_on < pd.Timestamp('now').date():
            print "Don't have any new papers for today, latest are from {}".format(published_on)
            return

        else:
            in_reply_to = None
            api = get_tweepy_api()
            print 'Tweeting {} papers published on {}'.format(len(to_tweet), published_on)
            for t in to_tweet:
                last_tweet = api.update_status(t, in_reply_to)
                in_reply_to = last_tweet.id
                sleep(TIME_BETWEEN_TWEETS)<commit_msg>prevent duplicate title tweet<commit_after>from time import sleep

import pandas as pd
import tweepy

from constants import MAX_TWEET_LENGTH, TWEET_THRESHOLD, TIME_BETWEEN_TWEETS
from credentials import *
from predict import add_predictions_to_date
from preprocessing import load_arxiv_and_tweets


def get_title_tweet(date=None):
    if date is None:
        date = pd.Timestamp('now').date()
    return 'arXiv papers, {}:'.format(date.strftime('%B %-m, %Y'))


def tweet_latest_day(dry_run=True):
    df = load_arxiv_and_tweets()
    index_to_predict_for = get_published_on_day_index(df)
    df = add_predictions_to_date(df, index_to_predict_for)
    predicted_papers = df.loc[index_to_predict_for]
    to_tweet = predicted_papers[predicted_papers.prediction > TWEET_THRESHOLD]
    if not to_tweet.empty:
        published_on = to_tweet.iloc[0].published.date()
        paper_tweets = to_tweet.sort_values('prediction', ascending=False).apply(get_tweet_text, axis=1)
        title_tweet = get_title_tweet()
        to_tweet = [title_tweet] + list(paper_tweets.values)

        if dry_run:
            for t in to_tweet:
                print t
                print

        elif published_on < pd.Timestamp('now').date():
            print "Don't have any new papers for today, latest are from {}".format(published_on)
            return

        else:
            in_reply_to = None
            api = get_tweepy_api()
            print 'Tweeting {} papers published on {}'.format(len(to_tweet), published_on)
            for t in to_tweet:
                last_tweet = api.update_status(t, in_reply_to)
                in_reply_to = last_tweet.id
                sleep(TIME_BETWEEN_TWEETS)
{'fraction_non_alphanumeric': 0.08238123872519543, 'fraction_numerical': 0.0012026458208057728, 'mean_word_length': 3.003610108303249, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1824186', 'n_tokens_mistral': 1141, 'n_tokens_neox': 1086, 'n_words': 230}