<commit_before>import logging
import argparse
from distutils.dir_util import mkpath
import time
import praw
from crawler_utils import save_submission


# Downloads all the self posts from given subreddit
# TODO: dynamically change ts_interval
def download_the_whole_subreddit(storage_dir, subreddit_name, ts_interval):
    mkpath(storage_dir, mode=0755)
    r = praw.Reddit(user_agent='SearchingReddit project 0.2 by /u/godlikesme')
    now_timestamp = int(time.time())
    cts2 = now_timestamp + 12*3600
    cts1 = now_timestamp - ts_interval
    # TODO start_timestamp = subreddit creation time
    while True:
        # get submissions here
        search_results = list(r.search('timestamp:{}..{}'.format(cts1, cts2), subreddit=subreddit_name, syntax='cloudsearch'))
        logging.debug("Got {} submissions in interval {}..{}".format(len(search_results), cts1, cts2))
        for submission in search_results:
            save_submission(submission, storage_dir)

        cts2 = cts1
        cts1 = cts2 - ts_interval


def main():
    logging.getLogger().setLevel(logging.DEBUG)

    parser = argparse.ArgumentParser(description='Download the whole subreddit')
    parser.add_argument("--storage_dir", dest="storage_dir", required=True)
    parser.add_argument("--subreddit", dest="subreddit", required=True, help="Download the whole subreddit")
    parser.add_argument("--timestamp_interval", dest="timestamp_interval", type=int, required=True)
    args = parser.parse_args()
    
    download_the_whole_subreddit(args.storage_dir, args.subreddit, args.timestamp_interval)

if __name__ == "__main__":
    main()
<commit_msg>Add largest_interval option to continue scraping from the previous run<commit_after>import logging
import argparse
from distutils.dir_util import mkpath
import time
import praw
from crawler_utils import save_submission

# Downloads all the self posts from given subreddit
# TODO: dynamically change ts_interval
def download_the_whole_subreddit(storage_dir, subreddit_name, ts_interval, largest_timestamp):
    mkpath(storage_dir, mode=0755)
    r = praw.Reddit(user_agent='SearchingReddit project 0.2 by /u/godlikesme')
    if largest_timestamp is None:
        largest_timestamp = int(time.time()) + 12*3600
    cts2 = largest_timestamp
    cts1 = largest_timestamp - ts_interval
    # TODO start_timestamp = subreddit creation time
    while True:
        # get submissions here
        search_results = list(r.search('timestamp:{}..{}'.format(cts1, cts2), subreddit=subreddit_name, syntax='cloudsearch'))
        logging.debug("Got {} submissions in interval {}..{}".format(len(search_results), cts1, cts2))
        for submission in search_results:
            save_submission(submission, storage_dir)

        cts2 = cts1
        cts1 = cts2 - ts_interval


def main():
    logging.getLogger().setLevel(logging.DEBUG)

    parser = argparse.ArgumentParser(description='Download the whole subreddit')
    parser.add_argument("--storage_dir", dest="storage_dir", required=True)
    parser.add_argument("--subreddit", dest="subreddit", required=True, help="Download the whole subreddit")
    parser.add_argument("--timestamp_interval", dest="timestamp_interval", type=int, required=True)
    parser.add_argument("--largest_timestamp", dest="largest_timestamp", type=int, required=False, default=None)
    args = parser.parse_args()
    
    download_the_whole_subreddit(args.storage_dir, args.subreddit, args.timestamp_interval, args.largest_timestamp)

if __name__ == "__main__":
    main()

{'fraction_non_alphanumeric': 0.10408394781622235, 'fraction_numerical': 0.012478729438457176, 'mean_word_length': 4.61624203821656, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3597672', 'n_tokens_mistral': 1069, 'n_tokens_neox': 1057, 'n_words': 244}