<jupyter_start><jupyter_text>Hashtags<jupyter_code>from nltk.tokenize import TweetTokenizer
import os
import pandas as pd
import re
import sys
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation
from IPython.display import clear_output
def squeal(text=None):
    clear_output(wait=True)
    if not text is None: print(text)
DATADIR = "../data/text/"
ID_STR = "id_str"
TEXT = "text"
TOPICQUERY = "corona|covid|huisarts|mondkapje|rivm|blijfthuis|flattenthecurve|houvol"
PANDEMICQUERY = "|".join([TOPICQUERY, r'virus|besmet|ziekenhui|\bic\b|intensive.care|^zorg|vaccin|[^ad]arts|uitbraak|uitbrak|pandemie|ggd|'+
                                      r'mondkapje|quarantaine|\bwho\b|avondklok|variant|verple|sympto|e.golf|mutant|^omt$|umc|hcq|'+
                                      r'hydroxychloroquine|virolo|zkh|oversterfte|patiënt|patient|intensivist|🦠|ivermectin'])
DISTANCEQUERY = "1[.,]5[ -]*m|afstand.*hou|hou.*afstand|anderhalve[ -]*meter"
LOCKDOWNQUERY = "lock.down|lockdown"
VACCINQUERY = "vaccin|ingeënt|ingeent|inent|prik|spuit|bijwerking|-->|💉|pfizer|moderna|astrazeneca|astra|zeneca|novavax|biontech"
TESTQUERY = r'\btest|getest|sneltest|pcr'
QUERY = "|".join([PANDEMICQUERY, TESTQUERY, VACCINQUERY, LOCKDOWNQUERY, DISTANCEQUERY])
BASEQUERY = "corona|covid"
HAPPY_QUERY = r'\b(geluk|gelukkig|gelukkige|blij|happy)\b'
LONELY_QUERY = r'eenza|alleen.*voel|voel.*alleen|lonely|loneli'
IK_QUERY = r'\b(ik|mij|mijn|me|mn|m\'n|zelf|mezelf|mijzelf|i)\b'
def get_tweets(file_pattern, query, query2="", spy=False):
    tweets = []
    file_names = sorted(os.listdir(DATADIR))
    for file_name in file_names:
        if re.search('^' + file_pattern, file_name):
            if spy:
                squeal(file_name)
            df = pd.read_csv(DATADIR+file_name,index_col=ID_STR)
            if query2 == "":
                df_query = df[df[TEXT].str.contains(query, flags=re.IGNORECASE)]
            else:
                df_query = df[df[TEXT].str.contains(query, flags=re.IGNORECASE) & df[TEXT].str.contains(query2, flags=re.IGNORECASE)]
            tweets.extend(list(df_query[TEXT]))
    return(tweets)
def get_hashtags(tweet):
    hashtags = []
    for token in TweetTokenizer().tokenize(tweet):
        if re.search(r'#', token):
            hashtags.append(token)
    return(hashtags)
def process_month(month, query=BASEQUERY, query2=""):
    tweets = [re.sub(r'\\n', ' ', tweet) for tweet in get_tweets(month, query, query2=query2, spy=False)]
    hashtags = {}
    for tweet in tweets:
        if re.search(r'#', tweet):
            for hashtag in get_hashtags(tweet):
                if hashtag in hashtags:
                    hashtags[hashtag] += 1
                else:
                    hashtags[hashtag] = 1
    print(month, " ".join([hashtag for hashtag in sorted(hashtags.keys(), key=lambda hashtag:hashtags[hashtag], reverse=True)][:200]))
pd.DataFrame([{"202105": "measures", "202106": "pandemic", "202107": "measures", 
               "202108": "pandemic", "202109": "entry pass", "202110": "entry pass"},
              {"202105": "pandemic", "202106": "measures", "202107": "pandemic", 
               "202108": "measures", "202109": "measures", "202110": "measures"},
              {"202105": "vaccination", "202106": "vaccination", "202107": "vaccination", 
               "202108": "vaccination obligation", "202109": "vaccination obligation", "202110": "pandemic"},
              {"202105": "entry pass", "202106": "FVD", "202107": "vaccination obligation", 
               "202108": "vaccination", "202109": "pandemic", "202110": "vaccination obligation"},
              {"202105": "Netherlands", "202106": "Netherlands", "202107": "FVD", 
               "202108": "Netherlands", "202109": "press conference", "202110": "press conference"},
              {"202105": "testing", "202106": "facemasks", "202107": "Netherlands", 
               "202108": "lockdown", "202109": "FVD", "202110": "unvaccinated"},
              {"202105": "FVD", "202106": "entry pass", "202107": "lockdown", 
               "202108": "press conference", "202109": "<NAME>", "202110": "3 October protest"},
              {"202105": "ivermectine", "202106": "app", "202107": "press conference", 
               "202108": "entry pass", "202109": "hospitality business", "202110": "Netherlands"},
              {"202105": "long covid", "202106": "variants", "202107": "long covid", 
               "202108": "FVD", "202109": "<NAME>", "202110": "<NAME>"},
              {"202105": "lockdown", "202106": "lab leak", "202107": "<NAME>", 
               "202108": "long covid", "202109": "<NAME>", "202110": "FVD"},
             ])
for month in "202105 202106 202107 202108 202109 202110".split():
    process_month(month)
for month in "202105".split():
    tweets = [re.sub(r'\\n', ' ', tweet) for tweet in get_tweets(month, LONELY_QUERY, query2=IK_QUERY, spy=False)]
    hashtags = {}
    for tweet in tweets:
        if re.search(r'#', tweet):
            for hashtag in get_hashtags(tweet):
                if hashtag in hashtags:
                    hashtags[hashtag] += 1
                else:
                    hashtags[hashtag] = 1
    print(month, " ".join([hashtag for hashtag in sorted(hashtags.keys(), key=lambda hashtag:hashtags[hashtag], reverse=True)][:200]))
for month in "202002 202003 202004 202005 202006 202007 202008 202009 202010 202011 202012 201201".split():
    tweets = [re.sub(r'\\n', ' ', tweet) for tweet in get_tweets(month, BASEQUERY, spy=False)]
    hashtags = {}
    for tweet in tweets:
        if re.search(r'#', tweet):
            for hashtag in get_hashtags(tweet):
                if hashtag in hashtags:
                    hashtags[hashtag] += 1
                else:
                    hashtags[hashtag] = 1
    print(month, " ".join([hashtag for hashtag in sorted(hashtags.keys(), key=lambda hashtag:hashtags[hashtag], reverse=True)][:200]))
for month in "202101 202102 202103 202104".split():
    tweets = [re.sub(r'\\n', ' ', tweet) for tweet in get_tweets(month, BASEQUERY, spy=False)]
    hashtags = {}
    for tweet in tweets:
        if re.search(r'#', tweet):
            for hashtag in get_hashtags(tweet):
                if hashtag in hashtags:
                    hashtags[hashtag] += 1
                else:
                    hashtags[hashtag] = 1
    print(month, " ".join([hashtag for hashtag in sorted(hashtags.keys(), key=lambda hashtag:hashtags[hashtag], reverse=True)][:200]))<jupyter_output>202101 #coronamaatregelen #coronavirus #corona #avondklok #coronadebat #COVID19 #lockdown #Corona #FVD #coronavaccin #COVID19NL #coronaprotest #Rutte #covid19nl #vaccin #covid19 #avondklokrellen #persconferentie #vaccinatie #Covid19 #coronabeleid #Vaccinatie #rellen #museumplein #avondklokdebat #covid #stopdelockdown #Eindhoven #coronacrisis #toeslagenaffaire #Covid_19 #PVV #vaccinatiestrategie #Amsterdam #AlleenSamen #AvondklokProtest #dtv #CoronaCrisis #politie #Wilders #vaccineren #vrijheid #lockdown2021 #ikwildieprik #VVD #jinek #COVID #op1 #RIVM #OMT #coronarellen # #zorg #kabinet #rutte #mondkapjes #hugodejonge #Nederland #Urk #TheGreatReset #StemNederlandTerug #peiling #virus #Op1npo #ikdoenietmeermee #StemZeWeg #Ivermectine #samentegencorona #Nieuwsuur #coronavaccinatie #coronazwendel #opendebat #coronavirusnl #COVIDー19 #amsterdam #Pfizer #Coronavirus #Avondklok #D66 #coronanl #Vaccin #peilingen #ZeroCovid #onderwijs #blijfthuis #IkPrikHetNiet #LongCovid #vaccins #nosjournaal #[...]
{'fraction_non_alphanumeric': 0.15091584815503054, 'fraction_numerical': 0.07379877886912663, 'mean_word_length': 3.5119760479041915, 'pattern_counts': {'":': 61, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '2746', 'license': 'Apache-2.0', 'n_tokens_mistral': 3171, 'n_tokens_neox': 2673, 'n_words': 544, 'path': 'hashtags.ipynb', 'repo_name': 'puregome/notebooks', 'size': '7564'}