<jupyter_start><jupyter_text>Fetching Twitter dataThis is a simple notebook containing a simple demonstration on how to fetch tweets using a Twitter Sanbox Environment. The sample data is saved in the form of a json file, which must then be preprocessed.<jupyter_code>import os
from os.path import join
from searchtweets import load_credentials, gen_rule_payload, ResultStream, collect_results
import json

project_dir = join(os.getcwd(), os.pardir)
raw_dir = join(project_dir, 'data', 'raw')
twitter_creds_path = join(project_dir, 'twitter_creds.yaml')
search_args = load_credentials(twitter_creds_path, yaml_key='sample_tweets_api')

# this should probably be moved to the configs.yaml
query = "((cyclone amphan) OR amphan)"


##Cyclone amphan
#Formed:16 May 2020
#Dissipated:21 May 2020

from_date="2020-05-14"
to_date="2020-06-15"

# I defined results_per_call as 100 which is default for free users. These can be 500 for paid tiers.
rule = gen_rule_payload(query, results_per_call=100, from_date="2020-05-14", to_date="2020-06-15")

rs = ResultStream(rule_payload=rule,
                  max_results=200,
                  **search_args)
fname = f'SAMPLE_DATA_QUERY_{query}_FROMDATE_{from_date}_TODATE_{to_date}.json'
with open(join(raw_dir, fname), 'a', encoding='utf-8') as f:
    for tweet in rs.stream():
        json.dump(tweet, f)
        f.write('\n')
print('done')<jupyter_output>done<jupyter_text>Count existing tweets for a given request<jupyter_code>search_args = load_credentials(twitter_creds_path, yaml_key='search_tweets_api')
query = "(cyclone amphan)"

count_rule = gen_rule_payload(query, from_date="2020-05-14", to_date="2020-06-15", count_bucket="day", results_per_call=500)

counts = collect_results(count_rule, result_stream_args=search_args)

counts
tweets = 0
for day in counts:
    tweets+=day['count']

tweets<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10481283422459893, 'fraction_numerical': 0.041176470588235294, 'mean_word_length': 5.828467153284672, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '668288', 'license': 'MIT', 'n_tokens_mistral': 702, 'n_tokens_neox': 636, 'n_words': 161, 'path': 'notebooks/1.0-jf-fetching-tweets-example.ipynb', 'repo_name': 'joaopfonseca/solve-iwmi', 'size': '1870'}