<jupyter_start><jupyter_text>Coverage tests for collected tweetsEstimate what percentage of Dutch tweets are collected by twiqs.nl<jupyter_code>import math
import os
import pandas as pd
import re
import sys
import multiprocessing as mp
from IPython.display import clear_output
DATADIRTEXT = "/home/erikt/projects/puregome/data/text/"
DATADIRCLOUD = "/home/erikt/projects/puregome/data/text-cloud/"
def squeal(text=None):
    clear_output(wait=True)
    if not text is None: print(text)<jupyter_output><empty_output><jupyter_text>Extimate coverage with reply ids<jupyter_code>IDSTR = "id_str"
INREPLYTOSTATUSIDSTR = "in_reply_to_status_id_str"

def getReplyPercentage(filePattern):
    files = sorted(os.listdir(DATADIRTEXT))
    found = 0
    missing = 0
    seenIds = {}
    for inFileName in files:
        if re.search(filePattern,inFileName):
            squeal(inFileName)
            df = pd.read_csv(DATADIRTEXT+inFileName,dtype={IDSTR:object})
            seenIds = {**seenIds,**{idStr:True for idStr in df[IDSTR]}}
    files = sorted(os.listdir(DATADIRCLOUD))
    for inFileName in files:
        if re.search(filePattern,inFileName):
            squeal(inFileName)
            df = pd.read_csv(DATADIRCLOUD+inFileName,dtype={INREPLYTOSTATUSIDSTR:object})
            for idStr in df[INREPLYTOSTATUSIDSTR]:
                if idStr == idStr:
                    if idStr in seenIds: found += 1
                    else: missing += 1
    return(found/(found+missing))
filePattern = "202007"
percentage = getReplyPercentage(filePattern)
print(f"target date: {filePattern}; percentage: {percentage}")<jupyter_output>20200726-08.out.gz<jupyter_text>Query token language coverageHow many of the tweets containing a query token are written in Dutch?<jupyter_code>DATADIRCOVERAGE = "./"
FILEPATTERNCOVERAGE = "20200701-0[0-57-9]|20200701-1[0-7]|20200701-2[12]"
TEXT = "text"
LANG = "lang"
DUTCH = "dutch"
OTHER = "other"
UNKNOWN = "unknown"

def coverage(token):
    files = sorted(os.listdir(DATADIRCOVERAGE))    
    langs = {}
    for inFileName in files:
        if re.search(FILEPATTERNCOVERAGE,inFileName):
            df = pd.read_csv(inFileName,compression="gzip")
            for i in range(0,len(df)):
                try:
                    if re.search(r'\b'+token+r'\b',df.iloc[i][TEXT]):
                        lang = df.iloc[i][LANG]
                        if lang in langs: langs[lang] += 1
                        else: langs[lang] = 1
                except: pass
    return(langs)

def coverageDutchOld(token):
    langs = coverage(token)
    summary = {DUTCH:0,OTHER:0}
    for lang in langs:
        if lang == DUTCH: summary[DUTCH] += langs[lang]
        elif lang != UNKNOWN: summary[OTHER] += langs[lang]
    if summary[DUTCH] == 0: return(0)
    else: return(summary[DUTCH]/(summary[DUTCH]+summary[OTHER]))
    
def coverageDutch(token,tweetsLang):
    summary = {DUTCH:0,OTHER:0}
    for text in tweetsLang.keys():
        if token in text.split():
            if tweetsLang[text] == DUTCH: summary[DUTCH] += 1
            elif tweetsLang[text] != UNKNOWN: summary[OTHER] += 1
    if summary[DUTCH] == 0: return(0)
    else: return(summary[DUTCH]/(summary[DUTCH]+summary[OTHER]))<jupyter_output><empty_output><jupyter_text>Find best keywords<jupyter_code>from nltk.tokenize import TweetTokenizer
SKIPTOKENS = "rt amsterdam the woof rosé de en in is van me ben we via nos juno br staysafe spnmarais arcinho kamafotos youtube \
update amp bts_twt blackpink baby and omg best ygofficialblink on ig goal wonderland ten instagram top you wonwoo daniel \
1advancehbdmaheshbabu with open of for vlive ver to netherlands jungkook up joonie jennie from super my kpop klopp gt vlog queen \
nct love by boy bergwijn zoe twitter teen te stop so more master lee hahahaha gogh god ever bruyne an ameen tik this stream slap \
school one oh ni man level ko just im hi here help he duit deluxe day da better beef be art aameen zico yes wtf wtaf winner will \
vn video vc total that team superior street snoop re pubg pink oopsie no new ne namkook moots moon miss maam look link la ka juliet \
jinnie jinkook his hey hahahahaha goals go gemes fan een dus do black back am all aaron zombie yep yeah woop won win weekend war un \
tingz thank tekken taekook sterling star stan sorry snap sex sb19official robin poor pls out onvres onde omfg ok ns now not need \
moonbin mood min mein maap maaf live let legend left laptop ke kangen jimin jan it iq how hot hoodie holy her hee happy hahahahah \
haha ha groningen got gg gente funny ft ff fc engineer eh ee down donde dm damn cute chief can bro big bestie banget at as \
xiaojun gee advancehbdmaheshbabu weareoneexo kang tier straykids kun dream tiktok than kevin jin haechan golden boys actorvijay \
unnie tweets pop only mochi missing like genie don bts alice it's he's i'm let's zone tok she mo made kids howyoulikethat era el \
duo drop angel allah urstrulymahesh winwin exo official dance hours heels has el edit proud".split()
SKIPTOKENS = r", . ! ? : ; - ( ) @ # $ / [ ] < >  ' + = | .. ... ` & \" ".split()
SKIPTOKENS = "😉 ° 🥇 🥈 🥉 – ¿ ¥ ☀ wonderland".split()
FILEPATTERN = "202006"
TEXT = "text"
BASELEXICON = "".split()
ALLCOUNT = 596893 # 5623571 # 22798953 # 22798953
LEFT200 = 61599
LEFT100 = 80258 # 349598 # 1724488 # 1231745
LEFT200100 = 43622 # 179089 # 801171 # 732526
LEFT300200100 = 34583 # 130887 # 557205 # 531315
LEFT400300200100 = 29528 # 109099 # 451231 # 439565
LEFT500400300200100 = 0 # # 394609 # 387447
LEFT600500400300200100 = 0 # # 357003 # 352026
LEFT700600500400300200100 = 0 # # 329054 # 324628
LEFT800700600500400300200100 = 0 # # 306745 # 303422

results = []

def cleanupText(text):
    text = re.sub(r"\\n"," ",text.lower())
    text = re.sub(r"[#@]"," ",text)
    return(text)

def tokenize(tokenizer,text):
    tokens = tokenizer.tokenize(text)
    return(" ".join(sorted(list(set(tokens)))))

def cleanupTokens(text):
    tokens = set(text.split())
    newTokens = []
    for token in tokens:
        if not re.search(r"http",token) and not token in SKIPTOKENS and len(token) > 1: newTokens.append(token)
    return(" ".join(newTokens))

def getTweetsFile(inFileName,dataDir):
    tweets = {}
    squeal(inFileName)
    tokenizer = TweetTokenizer()
    df = pd.read_csv(dataDir+inFileName)
    for i in range(0,len(df)):
        text = cleanupText(df.iloc[i][TEXT])
        text = tokenize(tokenizer,text)
        text = cleanupTokens(text)
        if LANG in df: tweets[text] = df.iloc[i][LANG]
        else: tweets[text] = True
    return(tweets)

def collectResult(result):
    global results
    results.append(result)

def getTweetsParallel(filePattern=FILEPATTERN,dataDir=DATADIR):
    global results
    files = sorted(os.listdir(dataDir))
    inFileNames = []
    for inFileName in files:
        if re.search(filePattern,inFileName): inFileNames.append(inFileName)
    pool = mp.Pool(mp.cpu_count())
    results = [pool.apply(getTweetsFile,args=[inFileName,dataDir]) for inFileName in inFileNames]
    pool.close()
    while len(results) > 1:
        toBeDeleted = []
        for i in range(0,len(results),2):
            if i < len(results)-1: 
                results[i] = {**results[i],**results[i+1]}
                toBeDeleted.append(i+1)
        toBeDeleted.reverse()
        for i in toBeDeleted: del(results[i])
    return(results[0])

def getTweets(filePattern=FILEPATTERN,dataDir=DATADIR):
    files = sorted(os.listdir(dataDir))
    tokenizer = TweetTokenizer()
    tweets = {}
    for inFileName in files:
        if re.search(filePattern,inFileName):
            squeal(inFileName)
            df = pd.read_csv(dataDir+inFileName)
            for i in range(0,len(df)):
                text = cleanupText(df.iloc[i][TEXT])
                text = tokenize(tokenizer,text)
                text = cleanupTokens(text)
                if LANG in df: tweets[text] = df.iloc[i][LANG]
                else: tweets[text] = True
    return(tweets)
tweetsLangOrg = getTweets(filePattern=FILEPATTERNCOVERAGE,dataDir=DATADIRCOVERAGE)
tweetsOrg = getTweetsParallel()
tweetsLang = dict(tweetsLangOrg)
tweets = dict(tweetsOrg)
lexicon = {token:0 for token in list(BASELEXICON)}
skipTokens = {s:0 for s in SKIPTOKENS}
processedTweetCounts = []
THRESHOLD = 0.9
MAXLEXICONSIZE = 800

coverageDutchScore = ""
while len(lexicon) < MAXLEXICONSIZE:
    frequencies = {}
    processedTweetCount = 0
    counter = 0
    toBeDeleted = []
    if len(lexicon) > 0: lastToken = list(lexicon.keys())[-1]
    else: lastToken = ""
    for text in tweets.keys():
        counter += 1
        if counter%10000 == 0: squeal(str(counter)+"/"+str(len(tweets))+" "+str(len(lexicon))+" "+lastToken+" "+str(coverageDutchScore))
        tokens = text.split()
        if len(set(lexicon.keys()).intersection(tokens)) >= 1:
            toBeDeleted.append(text)
        else:
            processedTweetCount += 1
            for token in tokens:
                if token in frequencies: frequencies[token] += 1
                else: frequencies[token] = 1
    for text in toBeDeleted:
        del(tweets[text])
    #toBeDeletedLang = []
    #for text in tweetsLang.keys():
    #    tokens = text.split()
    #    if len(set(lexicon).intersection(tokens)) >= 1:
    #        toBeDeletedLang.append(text)
    #for text in toBeDeletedLang:
    #    del(tweetsLang[text])    
    processedTweetCounts.append(processedTweetCount)
    frequencies = {k:frequencies[k] for k in sorted(frequencies.keys(),key=lambda k:frequencies[k],reverse=True)}
    for token in frequencies:
        if not token in skipTokens and len(token) > 1:
            coverageDutchScore = coverageDutch(token,tweetsLang)
            squeal(str(len(lexicon))+" "+str(processedTweetCount)+" "+str(len(skipTokens))+" "+str(frequencies[token])+" "+str(coverageDutchScore)+" "+token)
            if coverageDutchScore >= THRESHOLD:
                lexicon[token] = coverageDutchScore
                break
            else: skipTokens[token] = coverageDutchScore
    squeal(str(len(lexicon))+" "+str(processedTweetCount)+" "+str(len(skipTokens))+" "+str(frequencies[list(lexicon.keys())[-1]])+" "+list(lexicon.keys())[-1]+" "+str(coverageDutchScore))
    
" ".join(list(lexicon.keys()))
" ".join(list(lexicon.keys()))
for token in lexicon:
    if len(token) < 2: print(token)
for token in list(skipTokens.keys())[:20]:
    print(skipTokens[token],token)
outFile = open("tmp.txt","w")
print("track=",end="",file=outFile)
print(",".join([lexicon[i] for i in range(0,len(lexicon)) if i % 4 == 0 or i % 4 == 3]),file=outFile)
print("track=",end="",file=outFile)
print(",".join([lexicon[i] for i in range(0,len(lexicon)) if i % 4 == 1 or i % 4 == 2]),file=outFile)
outFile.close()
print(" ".join([lexicon[i] for i in range(0,len(lexicon)) if i % 4 == 0 or i % 4 == 3]))
print(" ".join([lexicon[i] for i in range(0,len(lexicon)) if i % 4 == 1 or i % 4 == 2]))
tweetsCopy = tweets
for i in range(0,len(lexicon)):
    if i % 100 == 0: print(i,len(tweetsCopy))
    toBeDeleted = []
    for text in tweetsCopy.keys():
        tokens = text.split()
        if len(set([lexicon[i]]).intersection(tokens)) > 0:
            toBeDeleted.append(text)
    for text in toBeDeleted:
        del(tweetsCopy[text])
print(i,len(tweetsCopy))
print(" ".join([str(processedTweetCounts[i]) for i in range(0,800,100)]))
sortedFrequencies = {k:frequencies[k] for k in sorted(frequencies.keys(),key=lambda k:frequencies[k],reverse=True)}
topTokens800 = [k for k in sortedFrequencies.keys() if re.search(r"[a-z0-9]",k) or \
                                                       (len(k) > 1 and k != ".." and k != "..." or
                                                       (len(k) == 1 and ord(k) >= 127000 and ord(k) < 130000))][0:100]
print(",".join(topTokens100))
print(",".join(topTokens200))
print(",".join(topTokens300))
print(",".join(topTokens400))
print(",".join(topTokens500))
print(",".join(topTokens600))
print(",".join(topTokens700))
print(",".join(topTokens800))
for token in topTokens100+topTokens200+topTokens300+topTokens400: print(token,end=",")
print()<jupyter_output><empty_output><jupyter_text>Token frequencies<jupyter_code>THRESHOLDTEXTCAT = 0.99
MAXLEXICONTEXTCAT = 800

tweetsTextCat = dict(tweetsLangOrg)

tokenFreqs = {}
for text in tweetsTextCat.keys():
    if tweetsTextCat[text] == DUTCH:
        for token in text.split():
            if token in tokenFreqs: tokenFreqs[token] += 1
            else: tokenFreqs[token] = 1

#discarded = []
#selected = []
for token in sorted(tokenFreqs.keys(),key=lambda k:tokenFreqs[k],reverse=True)[807+len(discarded):]:
    coverageDutchScore = coverageDutch(token,tweetsTextCat)
    if coverageDutchScore < THRESHOLDTEXTCAT: 
        print("          ",end="")
        discarded.append(token)
    else:
        selected.append(token)
    print(len(selected),tokenFreqs[token],coverageDutch(token,tweetsTextCat),token)
    if len(selected) >= MAXLEXICONTEXTCAT: break
TOKENSIGNORE = "‘ … € 🍀 – ° blm corona covid én één etc euro z'n 1,5".split()

toBeDeleted = []
for i in range(len(selected)-1,-1,-1):
    if selected[i] in TOKENSIGNORE: toBeDeleted.append(i)
for i in toBeDeleted: del(selected[i])
for token in selected:
    if len(token) <= 2: print(token)
outFile = open("tmp.txt","w")
print("track=",end="",file=outFile)
print(",".join([selected[i] for i in range(0,len(selected)) if i % 4 == 0 or i % 4 == 3]),file=outFile)
print("track=",end="",file=outFile)
print(",".join([selected[i] for i in range(0,len(selected)) if i % 4 == 1 or i % 4 == 2]),file=outFile)
outFile.close()
print(" ".join([selected[i] for i in range(0,len(selected)) if i % 4 == 0 or i % 4 == 3]))
print(" ".join([selected[i] for i in range(0,len(selected)) if i % 4 == 1 or i % 4 == 2]))
tweetsTextCatNew = getTweets(filePattern="20200711",dataDir=DATADIRCOVERAGE)
for token in selected:
    coverageDutchScore = coverageDutch(token,tweetsTextCatNew)
    if coverageDutchScore < 0.95:
        print(round(coverageDutchScore,3),token)<jupyter_output>0.912 vast
0.85 nieuw
0.947 vrouwen
0.917 woord
0.926 zat
0.935 ter
0.895 idd
0.941 zag
0.941 reactie
0.931 gesprek
0.929 wilde
0.929 gevoel
0 congo
0.917 sowieso
0.929 cultuur
0.944 bestaan
0.889 trein
0.938 mond
0.923 waarheid
0.933 macht
0 zjosdekker
0 asscher
0.923 licht
0.909 vandaan
0 ananninga
0 afschaffing
0.857 tip
0 2ekamertweets
0.909 waarvan
{'fraction_non_alphanumeric': 0.1282979463974939, 'fraction_numerical': 0.04497041420118343, 'mean_word_length': 3.5232997481108312, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 1, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '124159', 'license': 'Apache-2.0', 'n_tokens_mistral': 5443, 'n_tokens_neox': 4988, 'n_words': 1256, 'path': 'coverage.ipynb', 'repo_name': 'puregome/notebooks', 'size': '14365'}