<commit_before>import json
import tweepy
import sys
import os.path
import argparse
import datetime


def observed_tweets(data_file_name):
    # get the set of tweets already in the data file to avoid duplicates
    tweet_ids = set()
    if os.path.isfile(data_file_name):
        with open(data_file_name, 'r') as data_file:
            for line in data_file:
                status = json.loads(line)
                tweet_ids.add(status['id'])
    else:
        print 'WARNING - Did not find an existing data file. Will create new file: ' + str(data_file_name)

    return tweet_ids


def main(args):

    # load config file with Twitter account details
    if os.path.isfile(args.config_file):
        with open(args.config_file, 'r') as config_file:
            config = json.load(config_file)
    else:
        raise ValueError('ERROR - config file not found:' + str(args.config_file))

    # check the data path and file
    data_path = args.data_path
    data_file_name = os.path.join(data_path, args.data_file)

    if not os.path.isdir(args.data_path):
        if not os.path.exists(args.data_path):
            # create the new path
            os.makedirs(args.data_path)
        else:
            # path exists, but it's not a directory
            raise ValueError('ERROR - data path (' + str(args.data_path) + ')already exists, but it is not a directory.')

    # get the latest tweets and add them to the data file
    # some tweets may be duplicates for ones recorded earlier, however, some data such as retweet count may be updated
    # giving some ability to see if the short term popularity of a tweet spikes<commit_msg>create a backup of the data file before running<commit_after>import json
import tweepy
import sys
import os.path
import argparse
import datetime
import shutil


def observed_tweets(data_file_name):
    # get the set of tweets already in the data file to avoid duplicates
    tweet_ids = set()
    if os.path.isfile(data_file_name):
        with open(data_file_name, 'r') as data_file:
            for line in data_file:
                status = json.loads(line)
                tweet_ids.add(status['id'])
    else:
        print 'WARNING - Did not find an existing data file. Will create new file: ' + str(data_file_name)

    return tweet_ids


def main(args):

    # load config file with Twitter account details
    if os.path.isfile(args.config_file):
        with open(args.config_file, 'r') as config_file:
            config = json.load(config_file)
    else:
        raise ValueError('ERROR - config file not found:' + str(args.config_file))

    # check the data path and file
    data_path = args.data_path
    data_file_name = os.path.join(data_path, args.data_file)

    if not os.path.isdir(args.data_path):
        if not os.path.exists(args.data_path):
            # create the new path
            os.makedirs(args.data_path)
        else:
            # path exists, but it's not a directory
            raise ValueError('ERROR - data path (' + str(args.data_path) + ')already exists, but it is not a directory.')

    # create backup of old file
    backup_file_name = data_file_name + '.bak'
    shutil.copy(data_file_name, backup_file_name)

    # get the latest tweets and add them to the data file
    # some tweets may be duplicates for ones recorded earlier, however, some data such as retweet count may be updated
    # giving some ability to see if the short term popularity of a tweet spikes
{'fraction_non_alphanumeric': 0.07634470792365529, 'fraction_numerical': 0.0, 'mean_word_length': 2.7313915857605178, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4161309', 'n_tokens_mistral': 1035, 'n_tokens_neox': 997, 'n_words': 365}