<jupyter_start><jupyter_text>* /reddit/comment/search* /reddit/submission/search* /reddit/subreddit/search<jupyter_code># Collection DATA with Pushshift API

with urllib.request.urlopen("https://api.pushshift.io/reddit/comment/search/?subreddit=bitcoin&after=1546318800&before=1564631940&sort_type=score&sort=desc&size=1000") as url:
    df_comment = pd.DataFrame(data = json.loads(url.read().decode())['data'])

with urllib.request.urlopen("https://api.pushshift.io/reddit/submission/search/?subreddit=bitcoin&after=1546318800&before=1564631940&sort_type=score&sort=desc&size=1000") as url:
    df = pd.DataFrame(data = json.loads(url.read().decode())['data'])


df.columns # author, created_utc, score, body, subreddit
def get_data(after, before):
    url = 'https://api.pushshift.io/reddit/comment/search/?size=1000&after='+str(after)+'&before='+str(before)+'&subreddit=bitcoin'
    with urllib.request.urlopen(url) as url:
        df = pd.DataFrame(data = json.loads(url.read().decode())['data'])[['created_utc','author','score','body','subreddit']]
    return df<jupyter_output><empty_output><jupyter_text>Use to convert time: https://www.unixtimestamp.com/index.phpJan- After(1/1/2019) 1546300800- Before(1/31/2019) 1548935999Feb- After(2/1/2019) 1548979200- Before(2/28/2019) 1551398399Mar- After(3/1/2019) 1551398400- Before(3/31/2019) 1554076799Apr- After(4/1/2019) 1554076800- Before(4/30/2019) 1556668799May- After(5/1/2019) 1556668800- Before(5/31/2019) 1559347199June- After(5/1/2019) 1559347200- Before(5/31/2019) 1561939199<jupyter_code>from datetime import date,timedelta,datetime,timestamp
import time

int(time.mktime(date(2019, 1, 1).timetuple()))

datetime.timestamp(date(2019, 1, 1))
def to_integer(dt_time):
    return 10000*dt_time.year + 100*dt_time.month + dt_time.day

date(2019, 1, 1)
end_date = date(2019, 8, 8)
to_integer(begin_date)
df = get_data(1559347200, 1561939199)
# Will run until all posts have been gathered 
# from the 'after' date up until before date
while len(df) > 0:
    # Calls get_data() with the created date of the last submission
    try:
        after = df['created_utc'].tail(1).item()
        df = df.append(get_data(after, 1561939199), ignore_index=True)
    except KeyError:
        break
        
print(len(df))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15585546364823682, 'fraction_numerical': 0.13670004353504572, 'mean_word_length': 5.660869565217391, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '327896', 'license': 'MIT', 'n_tokens_mistral': 1053, 'n_tokens_neox': 849, 'n_words': 139, 'path': 'dev/reddit_api.ipynb', 'repo_name': 'silvernine209/bitcoin_trader', 'size': '2297'}