<jupyter_start><jupyter_text>We used this file for getting tweet_ids related to the election in order to hydrate them.We used the data from 22 days before the election all the way to the election because 22 days before the election is the average start day of early voting (https://www.ncsl.org/research/elections-and-campaigns/early-voting-in-state-elections.aspx) and we wanted to classify tweets based on what people were saying around the time voting started to see if classifying these tweets could help predict state election winners.<jupyter_code>import pandas as pd
import numpy as np
date_list = []
for x in range (12,32):
    date_list.append("10/{}/20".format(x))
for x in range (1,4):
    date_list.append("11/{}/20".format(x)) 
print(date_list)
filename = "uselection_tweets_1jul_11nov.csv"
chunksize = 10 ** 6
df_main = pd.DataFrame()
for chunk in pd.read_csv(filename, chunksize=chunksize, sep=';', usecols=['Created-At', 'Id']):
    df = chunk
    df = df[(df['Created-At'].str.contains('|'.join(date_list)))]
    df_main = pd.concat([df_main,df])
#     display(df)
df_main.drop(columns=['Created-At'], inplace=True)
display(df_main)
def save_tweet_ids(number_of_tweets, filename):
    df_main_random = df_main.sample(n=number_of_tweets)
    election_tweet_ids_path = filename
    with open(election_tweet_ids_path, 'w') as f:
        for index,row in df_main_random.iterrows():
            f.write("%s\n" % str(row["Id"]))
save_tweet_ids(10000, "sample_10k_tweets.txt")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12071240105540897, 'fraction_numerical': 0.020448548812664908, 'mean_word_length': 5.043824701195219, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '48059', 'license': 'MIT', 'n_tokens_mistral': 537, 'n_tokens_neox': 509, 'n_words': 141, 'path': 'election_evaluation/get_election_tweet_ids.ipynb', 'repo_name': 'mayankshouche/DSLabFinalProject', 'size': '1516'}