<jupyter_start><jupyter_text>Fasttext tweet classification  Data file creation<jupyter_code>import datetime
import math
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import numpy as np
import os
import pandas as pd
import pipes
import random
import re
import tempfile
from nltk.tokenize import TweetTokenizer
from IPython.display import clear_output
DISTANCE = "distance"
FACEMASK = "mondkapje"
TOPIC = FACEMASK # <=======================================

DATADIR = "../data/"
if TOPIC == FACEMASK: FILETWEETS = TOPIC+"-tweets+nunl.csv"
else: FILETWEETS = TOPIC+"-tweets.csv"
FILEANNOTATIONS = FILETWEETS+".human-labels.txt"
FILEFASTTEXT = "fasttext-"+TOPIC+".csv"
LABELPREFIX = "__label__"
LARGEINT = 9999999999
LABEL = "label"
TEXT = "text"
USER = "user"
IDSTR = "id_str"
IRRELEVANT = "IRRELEVANT"
NEUTRAL = "NEUTRAL"
ANDERS = "ANDERS"
EENS = "EENS"
ONEENS = "ONEENS"
SUPPORTS = "SUPPORTS"
REJECTS = "REJECTS"
SOURCE = "source"
tweets = pd.read_csv(DATADIR+FILETWEETS,index_col=IDSTR)
annotations = pd.read_csv(DATADIR+FILEANNOTATIONS,header=None,sep=" ")
mainAnnotator = annotations.iloc[0][0]
def cleanup(text):
    text = re.sub(r"\\n"," ",text)
    text = re.sub(r"https://\S+","",text)
    text = re.sub(r"\s+"," ",text)
    text = text.strip()
    return(text)

def tokenize(text):
    return(" ".join(TweetTokenizer().tokenize(text)))

def preprocess(text):
    return(tokenize(cleanup(text)).lower())
fasttextData = {}
for i in range(0,len(annotations)):
    annotator = annotations.iloc[i][0]
    tweetId = annotations.iloc[i][2]
    if annotator == mainAnnotator and tweetId in list(tweets.index):
        tweetUser = tweets.loc[tweetId][1]
        tweetLabel = annotations.iloc[i][4]
        if tweetLabel == NEUTRAL: tweetLabel = IRRELEVANT
        if tweetLabel == ANDERS: tweetLabel = IRRELEVANT
        if tweetLabel == EENS: tweetLabel = SUPPORTS
        if tweetLabel == ONEENS: tweetLabel = REJECTS
        fasttextData[tweetId] = {LABEL:LABELPREFIX+tweetLabel,\
                                 USER:tweetUser,\
                                 TEXT:preprocess(tweets.loc[tweetId][TEXT])}
len(fasttextData),len(tweets),len(annotations)
outFile = open(FILEFASTTEXT,"w")
seenTexts = {}
for tweetId in fasttextData:
    text = cleanup(fasttextData[tweetId][TEXT])
    if not text in seenTexts:
        print(fasttextData[tweetId][LABEL],text,file=outFile)
        seenTexts[text] = True
outFile.close()
annotations1 = annotations[annotations[0]==mainAnnotator]
nbrOfAnnotationMinutes = len(set([str(x)[:12] for x in annotations1[1]]))
nbrOfAnnotatedTweets = len(set([str(x)[:12] for x in annotations1[2]]))
print("tweets annotated per minute:",round(nbrOfAnnotatedTweets/nbrOfAnnotationMinutes,1),"; 1000 tweets take:",\
      round(1000/(nbrOfAnnotatedTweets/nbrOfAnnotationMinutes)),"minutes")
annotations2 = annotations[annotations[0]!=mainAnnotator]
nbrOfAnnotationMinutes = len(set([str(x)[:12] for x in annotations2[1]]))
nbrOfAnnotatedTweets = len(set([str(x)[:12] for x in annotations2[2]]))
print("tweets annotated per minute:",round(nbrOfAnnotatedTweets/nbrOfAnnotationMinutes,1),"; 1000 tweets take:",\
      round(1000/(nbrOfAnnotatedTweets/nbrOfAnnotationMinutes)),"minutes")<jupyter_output>tweets annotated per minute: 1.0 ; 1000 tweets take: 1000 minutes<jupyter_text>Fasttext run and evaluation<jupyter_code>import fasttext
import random
from IPython.display import clear_output
PRETRAINEDDIR = "/home/erikt/projects/newsgac/fasttext-runs/"
WIKIFILENAME = "wiki.nl.vec"
def print_results(N, p, r):
    print("N\t" + str(N))
    print("P@{}\t{:.3f}".format(1, p))
    print("R@{}\t{:.3f}".format(1, r))
fasttextData = []
inFile = open(FILEFASTTEXT,"r")
for line in inFile: fasttextData.append(line.strip())
inFile.close()
len(fasttextData)
DIM = 100
EPOCH = 100
LR = 0.05
N = 10
TRAIN = "TRAIN"+str(int(random.random()*LARGEINT))
TEST = "TEST"+str(int(random.random()*LARGEINT))
VALIDATION = "VALIDATION"+str(int(random.random()*LARGEINT))
def ranSelect(inList,size):
    outList = []
    selectionList = list(inList)
    while len(outList) < size and len(selectionList) > 0:
        index = int(random.random()*len(selectionList))
        outList.append(selectionList[index])
        del(selectionList[index])
    return(outList)
def runFasttext(fasttextData,dim=DIM,epoch=EPOCH,lr=LR,n=N,squealFlag=True,maxTrain=None):
    predictionCountsTest = []
    predictionLabelsTest = []
    predictionCountsValidation = []
    predictionLabelsValidation = []
    for fold in range(0,n):
        if squealFlag:
            clear_output(wait=True)
            print("starting fold",fold)
        validationStart = round(fold*len(fasttextData)/n)
        validationEnd = round((fold+1)*len(fasttextData)/n)
        if fold < n-1: nextFold = fold+1
        else: nextFold = 0
        testStart = round(nextFold*len(fasttextData)/n)
        testEnd = round((nextFold+1)*len(fasttextData)/n)
        trainFile = open(TRAIN,"w")
        testFile = open(TEST,"w")
        validationFile = open(VALIDATION,"w")
        trainData = []
        validationData = []
        testData = []
        for i in range(0,len(fasttextData)):
            data = fasttextData[i]
            if i >= testStart and i < testEnd: 
                print(data,file=testFile)
                testData.append(data)
            elif i >= validationStart and i < validationEnd: 
                print(data,file=validationFile)
                validationData.append(data)
            else: 
                print(data,file=trainFile)
                trainData.append(data)
        testFile.close()
        trainFile.close()
        validationFile.close()
        if not maxTrain == None and maxTrain < len(trainData):
            trainData = ranSelect(trainData,maxTrain)
            trainFile = open(TRAIN,"w")
            for i in range(0,len(trainData)):
                print(trainData[i],file=trainFile)
            trainFile.close()
        model = fasttext.train_supervised(TRAIN, dim=dim, epoch=epoch, lr=lr, pretrainedVectors="twiqs-model-2020.vec")
        predictionCountsValidation.append([*model.test(VALIDATION)])
        predictionLabelsValidation.append(model.predict(validationData))
        predictionCountsTest.append([*model.test(TEST)])
        predictionLabelsTest.append(model.predict(testData))
        print(fold, end=" ")
        evaluate(predictionCountsTest, predictionLabelsTest, fasttextData[testStart:testEnd])
        os.unlink(TRAIN)
        os.unlink(TEST)
        os.unlink(VALIDATION)
    if squealFlag:
        clear_output(wait=True)
        print("finished")
    return(predictionCountsValidation,predictionLabelsValidation,predictionCountsTest,predictionLabelsTest)
def makeConfusionMatrix(fasttextData,predictionLabels):
    goldLabels = pd.Series([fasttextData[i].split()[0] for i in range(0,len(fasttextData))])
    predictedLabels = pd.Series([x[0] for row in predictionLabels for x in row[0]])
    return(pd.crosstab(goldLabels,predictedLabels))
def evaluate(predictionCounts,predictionLabels,fasttextData,printResults=True):
    caseTotal = 0
    pTotal = 0
    rTotal = 0
    for i in range(0,len(predictionCounts)):
        caseTotal += predictionCounts[i][0]
        pTotal += predictionCounts[i][0]*predictionCounts[i][1]
        rTotal += predictionCounts[i][0]*predictionCounts[i][2]
    precision = round(pTotal/caseTotal,3)
    recall = round(rTotal/caseTotal,3)
    cf = makeConfusionMatrix(fasttextData,predictionLabels)
    for label in ['__label__REJECTS','__label__SUPPORTS','__label__IRRELEVANT']:
        if not label in cf: cf[label] = [0 for i in range(0,len(cf))]
    numberOfPredictedSupportLabels = sum([cf.iloc[i]['__label__SUPPORTS'] for i in range(0,len(cf))])
    numberOfGoldSupportLabels = sum(cf.loc['__label__SUPPORTS'])
    numberOfPredictedRejectLabels = sum([cf.iloc[i]['__label__REJECTS'] for i in range(0,len(cf))])
    numberOfGoldRejectLabels = sum(cf.loc['__label__REJECTS'])
    fraction = (numberOfPredictedRejectLabels/numberOfPredictedSupportLabels)/(numberOfGoldRejectLabels/numberOfGoldSupportLabels)
    if printResults: 
        print("cases: {0}; precision: {1:0.3f}; recall: {2:0.3f}; fraction: {3:0.3f}".format(caseTotal,precision,recall,fraction))
    return((precision,fraction))<jupyter_output><empty_output><jupyter_text>Grid search (skip)<jupyter_code>scoresValidation = {}
scoresTest = {}
for dim in [10,20,50,100,200,300]:
    for epoch in [10,20,50,100,200]:
        for lr in [0.05,0.1,0.2]:
            predictionCountsValidation,predictionLabelsValidation,predictionCountsTest,predictionLabelsTest = \
                runFasttext(fasttextData,dim=dim,epoch=epoch,lr=lr,squealFlag=False)
            key = ",".join([str(dim),str(epoch),str(lr)])
            print(f"{dim:3} {epoch:3} {lr:0.2f}",end=" validation: ")
            scoresValidation[key] = evaluate(predictionCountsValidation,predictionLabelsValidation,fasttextData)
            print(f"{dim:3} {epoch:3} {lr:0.2f}",end="       test: ")
            scoresTest[key] = evaluate(predictionCountsTest,predictionLabelsTest,fasttextData)
print("optimizing accuracy via validation data")
[(scoresValidation[k],k) for k in sorted(scoresValidation.keys(),\
                                         key=lambda k:scoresValidation[k][0],reverse=True)][0:10]
print("checking best accuracy for test data")
[(scoresTest[k],k) for k in sorted(scoresTest.keys(),\
                                         key=lambda k:scoresTest[k][0],reverse=True)][0:10]<jupyter_output><empty_output><jupyter_text>Facemasks: best validation accuracy 0.56 for '20,50,0.1', associated test accuracy 0.56 (5th score).Social distancing: best validation accuracy 0.65 for '300,10,0.2', associated test accuracy 0.65 (top score).<jupyter_code>print("optimizing fraction via validation data")
[(scoresValidation[k],k) for k in sorted(scoresValidation.keys(),
                                         key=lambda k:scoresValidation[k][1],reverse=True)][0:10]
print("checking fraction for test data")
[(scoresTest[k],k) for k in sorted(scoresTest.keys(),
                                         key=lambda k:scoresTest[k][1],reverse=True)][0:10]<jupyter_output><empty_output><jupyter_text>Facemasks: best validation ratio: 1.48 for '10,200,0.2'; associated test ratio: 1.70 (8th scsore)Social distancing: best validation ratio 0.88 for '200,200,0.2'; associated test accuracy 0.83 (top score).  Check relation accuracy and fraction (skip)<jupyter_code>THRESHOLD = 0.62

plt.figure(figsize=(6,3))
ax1 = plt.subplot(121)
x1 = [scoresTest[key][0] for key in scoresValidation]
y1 = [scoresTest[key][1] for key in scoresValidation]
ax1.scatter(x1,y1)
ax2 = plt.subplot(122)
x2 = [scoresTest[key][0] for key in scoresValidation if scoresTest[key][0] > THRESHOLD]
y2 = [scoresTest[key][1] for key in scoresValidation if scoresTest[key][0] > THRESHOLD]
ax2.scatter(x2,y2)
plt.xlabel("fraction")
plt.ylabel("accuracy")
plt.show()
np.corrcoef(x1,y1)[0][1],np.corrcoef(x2,y2)[0][1]
N = 5
scoresN = {} 
triplets = [(20,300,0.2),(50,200,0.2),(300,200,0.2),(20,200,0.05),(10,300,0.2)]

for triplet in triplets:
    dim,epoch,lr = triplet
    key = " ".join([str(dim),str(epoch),str(lr)])
    scoresN[key] = []
    for i in range(0,N): 
        predictionCounts,predictionLabels = runFasttext(fasttextData,dim=dim,epoch=epoch,lr=lr,squealFlag=False)
        print(dim,epoch,lr,end=" ")
        scoresN[key].append(evaluate(predictionCounts,predictionLabels,fasttextData))
scoresNaverage = {}
for key in scoresN: scoresNaverage[key] = round(np.average([x[1] for x in scoresN[key]]),3)
{key:scoresNaverage[key] for key in sorted(scoresNaverage.keys(),key=lambda k:scoresNaverage[k],reverse=True)}
for dim in [10]:
    for epoch in [900]:
        for lr in [0.2]:
            predictionCounts,predictionLabels = runFasttext(fasttextData,dim=dim,epoch=epoch,lr=lr,squealFlag=False)
            print(dim,epoch,lr,end=" ")
            key = ",".join([str(dim),str(epoch),str(lr)])
            scores[key] = evaluate(predictionCounts,predictionLabels,fasttextData)<jupyter_output><empty_output><jupyter_text>Run with best parameters (skip, except for first block)<jupyter_code>def setBestParameters():
    if TOPIC == DISTANCE:
        # social distancing (202008: 20,300,0.9) (202009a:20,10,0.05) (202009b:10,10,0.2) (202010:200,200,0.2) (emnlp:10,10,0.2)
        BESTDIM = 300 # 20210304: was 200
        BESTEPOCH = 200
        BESTLR = 0.2
    elif TOPIC == FACEMASK:
        # facemasks (202008: 10,900,0.2) (202010:10,200,0.2) (emnlp:300,10,0.2)
        BESTDIM = 10
        BESTEPOCH = 200
        BESTLR = 0.2
    else:
        print("unknown topic!",file=sys.stderr)
        sys.exit(1)
    return(BESTDIM,BESTEPOCH,BESTLR)
        
BESTDIM,BESTEPOCH,BESTLR = setBestParameters()
predictionCountsValidation,predictionLabelsValidation,predictionCountsTest,predictionLabelsTest = \
    runFasttext(fasttextData, dim=BESTDIM, epoch=BESTEPOCH, lr=BESTLR, squealFlag=False)
evaluate(predictionCountsValidation,predictionLabelsValidation,fasttextData)
evaluate(predictionCountsTest,predictionLabelsTest,fasttextData)
predictionCountsValidation,predictionLabelsValidation,predictionCountsTest,predictionLabelsTest = \
    runFasttext(fasttextData, dim=BESTDIM, epoch=BESTEPOCH, lr=BESTLR, squealFlag=False)
labelCountsGold = {}
for i in range(0,len(fasttextData)):
    label = fasttextData[i].split()[0]
    if label in labelCountsGold: labelCountsGold[label] += 1
    else: labelCountsGold[label] = 1
labelCountsGold
print("baseline accuracy:",round(max(labelCountsGold.values())/sum(labelCountsGold.values()),3))
labelCountsPredicted = {}
for i in range(0,len(predictionLabelsTest)):
    for label in predictionLabelsTest[i][0]:
        if label[0] in labelCountsPredicted: labelCountsPredicted[label[0]] += 1
        else: labelCountsPredicted[label[0]] = 1
labelCountsPredicted
print("total absolute deviation",sum([abs(labelCountsPredicted[l]-labelCountsGold[l]) for l in labelCountsGold]))
print("relative deviation per label:",\
      {l:round(abs(labelCountsPredicted[l]-labelCountsGold[l])/labelCountsGold[l],3) for l in labelCountsGold})
makeConfusionMatrix(fasttextData,predictionLabelsTest)<jupyter_output><empty_output><jupyter_text>For the mondkapje data, fasttext predicts 63% of the labels correctly without external dictionary and 66% with a Wikipedia dictionary (baseline: 46%). It overestimates the presence of negative labels and underestimates the level of positive and neutral labels. The amount of irrelevant labels is about right<jupyter_code>multiplicationFactors = {}
for label in labelCountsGold:
    multiplicationFactors[label] = labelCountsGold[label]/labelCountsPredicted[label]
multiplicationFactors<jupyter_output><empty_output><jupyter_text>For mondkapje tweets, the multiplication factor for the positive label is unrealistically high so we will not use these factors.<jupyter_code>predictedNumberOfLabels = 0
for label in labelCountsGold:
    predictedNumberOfLabels += multiplicationFactors[label]*labelCountsPredicted[label]
print(predictedNumberOfLabels,sum(labelCountsGold.values()))<jupyter_output><empty_output><jupyter_text>Relation training data size - accuracy (skip)<jupyter_code>N = 5
expResultsAll = {}
for trainingSize in [100,200,500,1000,2000,5000,10000]:
    expResultsTrainingSize = []
    for i in range(0,N):
        selection = ranSelect(fasttextData,trainingSize)
        predictionCountsValidation,predictionLabelsValidation,predictionCountsTest,predictionLabelsTest = \
            runFasttext(fasttextData,dim=BESTDIM,epoch=BESTEPOCH,lr=BESTLR,squealFlag=False,maxTrain=trainingSize)
        expResultsTrainingSize.append(evaluate(predictionCountsTest,predictionLabelsTest,fasttextData,printResults=False))
    average = (np.average([x[0] for x in expResultsTrainingSize]),np.average([x[1] for x in expResultsTrainingSize]))
    expResultsAll[len(selection)] = average
    print("{0:4} {1:0.3f} {2:0.3f}".format(len(selection),average[0],average[1]))
    if len(selection) >= len(fasttextData): break
del(expResultsAll[list(expResultsAll.keys())[-1]])
PLOTFILENAME = "training-size.png"

plt.figure(figsize=(16,6))

ax1 = plt.subplot(121)
plt.plot(list(expResultsAll.keys()),[x[0] for x in list(expResultsAll.values())])
plt.xscale("log")
plt.ylabel("accuracy")
plt.xlabel("training data size")
plt.title("Accuracy related to training size")

ax1 = plt.subplot(122)
plt.plot(list(expResultsAll.keys()),[x[1] for x in list(expResultsAll.values())])
plt.xscale("log")
plt.ylabel("fraction")
plt.xlabel("training data size")
plt.title("Fraction score related to training size")
plt.savefig(PLOTFILENAME)
plt.show()<jupyter_output><empty_output><jupyter_text>Classifying unlabeled tweets<jupyter_code>import fasttext
import matplotlib
import numpy as np
import os
import pandas as pd
import re
from nltk.tokenize import TweetTokenizer
BESTDIM,BESTEPOCH,BESTLR = setBestParameters()
BASEDIR = "/home/erikt/projects/puregome/data/"
DATADIRECTORY = BASEDIR+"text/"
DATADIRECTORYREDDIT = BASEDIR+"reddit/text/"
DATADIRECTORYNUNL = BASEDIR+"nunl/text/"
TWITTER = "twitter"
REDDIT = "reddit"
NUNL = "nunl"
SOURCES = [TWITTER,REDDIT,NUNL]
DATADIRECTORIES = {TWITTER:DATADIRECTORY,REDDIT:DATADIRECTORYREDDIT,NUNL:DATADIRECTORYNUNL}
FILEFASTTEXT = "fasttext-"+TOPIC+".csv"
LABELPREFIX = "__label__"
NEGATIVE = LABELPREFIX+"NEGATIVE"
POSITIVE = LABELPREFIX+"POSITIVE"
NEUTRAL = LABELPREFIX+"NEUTRAL"
IRRELEVANT = LABELPREFIX+"IRRELEVANT"
SUPPORTS = LABELPREFIX+"SUPPORTS"
REJECTS = LABELPREFIX+"REJECTS"
TEXT = "text"
USER = "user"
TOTAL = "total"
DIM = BESTDIM
EPOCH = BESTEPOCH
LR = BESTLR
def cleanup(text):
    text = re.sub(r"\\n"," ",text)
    text = re.sub(r"https://\S+","",text)
    text = re.sub(r"\s+"," ",text)
    text = text.strip()
    return(text)

def tokenize(text):
    return(" ".join(TweetTokenizer().tokenize(text)))

def preprocess(text):
    return(tokenize(cleanup(text)).lower())
SCREENNAME = "screenname"

def makeGrepCommandFromQuery(query):
    grepCommand = "grep"
    for orPart in query.split("|"):
        grepCommand += ' -e "'+orPart+'"'
    return(grepCommand)

def readData(datePattern,query,dataDirectory=DATADIRECTORY):
    fileList = sorted(os.listdir(dataDirectory))
    testData = []
    for inFileName in fileList:
        if re.search(datePattern,inFileName) and os.path.exists(dataDirectory+inFileName):
            try:
                fileData = pd.read_csv(dataDirectory+inFileName).drop_duplicates()
                matchedText = fileData[fileData[TEXT].str.contains(query,case=False)]
                matchedTextPreprocessed = matchedText[TEXT].apply(lambda x:preprocess(x))
                testData.extend(list(matchedTextPreprocessed))
            except:
                pass
    return(testData)

def readDataGeo(datePattern,query,dataDirectory=DATADIRECTORY):
    fileList = sorted(os.listdir(dataDirectory))
    testData = []
    locations = []
    for inFileName in fileList:
        if re.search(datePattern,inFileName) and os.path.exists(dataDirectory+inFileName):
            try:
                fileData = pd.read_csv(dataDirectory+inFileName).drop_duplicates()
                matchedText = fileData[fileData[TEXT].str.contains(query,case=False)]
                matchedTextPreprocessed = matchedText[TEXT].apply(lambda x:preprocess(x))
                testData.extend(matchedTextPreprocessed)
            except:
                pass
    return(testData)

def classify(datePattern,query,model,dataDirectory=DATADIRECTORY):
    testData = readData(datePattern,query,dataDirectory)
    predictedLabels = model.predict(testData)
    predictedGroups = pd.DataFrame(predictedLabels[0]).groupby(0).groups
    labelCountsPredicted = {label:len(predictedGroups[label]) for label in predictedGroups}
    nbrOfLabels = sum([labelCountsPredicted[label] for label in labelCountsPredicted if label != IRRELEVANT])
    labelPercentages = { label:round(100*labelCountsPredicted[label]/nbrOfLabels,1) for label in labelCountsPredicted}
    labelPercentages[TOTAL] = nbrOfLabels
    return(labelPercentages)
import ast


RESULTSFILE = "results-"+TOPIC+".csv"


def storeResults(results, results_file=RESULTSFILE):
     pd.DataFrame(results).T.to_csv(results_file, index_label=SOURCE)

        
def readResults(results_file=RESULTSFILE):
    try:
        df = pd.read_csv(results_file, index_col=SOURCE)
        for source in df.index:
            for date in df.loc[source].index:
                try:
                    df.loc[source][date] = ast.literal_eval(df.loc[source][date])
                except: pass
        return(removeNan(df.T.to_dict()))
    except:
        return({})


def combineResults(results_target, results_source):
    for source in results_source:
        if not source in results_target:
            results_target[source] = {}
        for date in results_source[source]:
            if date in results_target[source]:
                print(f"duplicate date: {date}!")
            else:
                results_target[source][date] = results_source[source][date]

            
def getTotals(results):
    totals = {}
    for source in results:
        for date in results[source]:
            try:
                if not source in totals: totals[source] = 0
                totals[source] += results[source][date][TOTAL]
            except: pass
    return(totals)


def removeNan(results):
    for source in results:
        toBeDeleted = []
        for date in results[source]:
            if results[source][date] != results[source][date]:
                toBeDeleted.append(date)
        for date in toBeDeleted: del(results[source][date])
    return(results)
results2021 = readResults()
results2020 = readResults(results_file=RESULTSFILE + ".20201231")
results = {}
combineResults(results, results2020)
combineResults(results, results2021)
totals =  getTotals(results)
MAXDAYSPERMONTH = 31
MODELFILE = f"model-{TOPIC}-{BESTDIM}-{BESTEPOCH}-{BESTLR}.bin"

if TOPIC == DISTANCE:
    QUERY = "1[.,]5[ -]*m|afstand.*hou|hou.*afstand|anderhalve[ -]*meter"
elif TOPIC == FACEMASK:
    QUERY = TOPIC
else:
    print("unknown topic!",file=sys.stderr)
    sys.exit(1)
MAXDAYSPERMONTH = 31

model = fasttext.train_supervised(FILEFASTTEXT,dim=BESTDIM,epoch=BESTEPOCH,lr=BESTLR)
model.save_model(MODELFILE)
model = fasttext.load_model(MODELFILE)<jupyter_output>Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.<jupyter_text>Recompute results per data source (run once per month)<jupyter_code>results = {}
totals = {}
for source in SOURCES:
    dataDirectory = DATADIRECTORIES[source]
    total = 0
    results[source] = {}
    for month in "202101 202102 202103".split():
        for day in range(1,MAXDAYSPERMONTH+1):
            date = month+str(day).zfill(2)
            try:
                labels = classify(date,QUERY,model,dataDirectory=dataDirectory)
                if not REJECTS in labels: labels[REJECTS] = 0
                if not SUPPORTS in labels: labels[SUPPORTS] = 0
                if not IRRELEVANT in labels: labels[IRRELEVANT] = 0
                if labels[REJECTS]+labels[SUPPORTS]+labels[IRRELEVANT] > 0:
                    results[source][date] = labels
                    print(source,date,labels[REJECTS],labels[SUPPORTS],labels[IRRELEVANT],labels[TOTAL])
                    total += labels[TOTAL]
            except:
                print(source,date,"error")
    totals[source] = total
    print("total relevant found:",total)
storeResults(results)<jupyter_output><empty_output><jupyter_text>Continue here...<jupyter_code>totals
for platform in "twitter reddit nunl".split():
    counts = {}
    for date in results[platform]:
        try:
            if TOTAL in results[platform][date] and results[platform][date][TOTAL] > 0:
                for label in results[platform][date].keys():
                    if not label in counts: counts[label] = 0
                    if label == TOTAL: counts[label] += results[platform][date][label]
                    else: counts[label] += round(results[platform][date][TOTAL]*results[platform][date][label]/100.0)
        except: pass
    print(platform,counts)<jupyter_output>twitter {'__label__IRRELEVANT': 771093, '__label__REJECTS': 807670, '__label__SUPPORTS': 148542, 'total': 956214}
reddit {'__label__IRRELEVANT': 2168, '__label__REJECTS': 1779, '__label__SUPPORTS': 667, 'total': 2446}
nunl {'__label__IRRELEVANT': 27364, '__label__REJECTS': 31795, '__label__SUPPORTS': 12017, 'total': 43812}<jupyter_text>Social distancing:```twitter {'__label__IRRELEVANT': 310420, '__label__REJECTS': 277596, '__label__SUPPORTS': 725624, 'total': 1003220}reddit {'__label__SUPPORTS': 2585, 'total': 3821, '__label__REJECTS': 1236, '__label__IRRELEVANT': 449}nunl {'__label__SUPPORTS': 40032, 'total': 53345, '__label__REJECTS': 13313, '__label__IRRELEVANT': 7647}```Face masks:```twitter {'__label__IRRELEVANT': 771093, '__label__REJECTS': 807670, '__label__SUPPORTS': 148542, 'total': 956214}reddit {'__label__IRRELEVANT': 2168, '__label__REJECTS': 1779, '__label__SUPPORTS': 667, 'total': 2446}nunl {'__label__IRRELEVANT': 27364, '__label__REJECTS': 31795, '__label__SUPPORTS': 12017, 'total': 43812}```<jupyter_code>df = pd.read_csv(RESULTSFILE,index_col="source").T
totalsIncludingIrrelevant = {source:0 for source in [TWITTER,REDDIT,NUNL]}
for i in range(0,len(df)):
    for source in [TWITTER,REDDIT,NUNL]:
        try:
            dictionary = ast.literal_eval(df.iloc[i][source])
        except:
            dictionary = {TOTAL:0}
        if dictionary[TOTAL] > 0:
            totalsIncludingIrrelevant[source] += round((1+dictionary[IRRELEVANT]/100)*dictionary[TOTAL])
totalsIncludingIrrelevant<jupyter_output><empty_output><jupyter_text>Visualization over time<jupyter_code>import datetime
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()
def movingAverage(numberList,size):
    newList = []
    start_offset = int(size/2)
    for i in range(0,len(numberList)):
        newList.append(numberList[i])
        numbersUsed = 1
        for j in range(-start_offset,size-start_offset):
            if i+j >= 0 and i+j < len(numberList) and i+j != i:
                newList[i] += numberList[i+j]
                numbersUsed += 1
        newList[i] /= numbersUsed
    return(newList)
def pretty_number(number):
    digits = str(number)
    pretty_digits = ""
    for i in range(0,len(digits)):
        if i > 0 and i % 3 == 0: pretty_digits = ","+pretty_digits
        pretty_digits = digits[-1-i]+pretty_digits
    return(pretty_digits)
DATEFORMAT = "%Y%m%d"
DAYSCOMBINED = 30
EXTRASMOOTHDAYS = 7
STARTDATE = "20200310"
LABELS = {TWITTER: f"Twitter ({pretty_number(totals[TWITTER])})",\
          NUNL: f"Nu.nl ({pretty_number(totals[NUNL])})",\
          REDDIT: f"Reddit ({pretty_number(totals[REDDIT])})"}
if TOPIC == DISTANCE:
    PLOTFILENAME = "social-distancing-all.png"
elif TOPIC == FACEMASK:
    PLOTFILENAME = "mondkapje-all.png"
else:
    print("unknown topic!",file=sys.stderr)
    sys.exit(1)

x = {}
ySupports = {}
plt.subplots(figsize=(12,6))
font = {"size":16}
matplotlib.rc("font",**font)
ax = plt.subplot(111)
for source in [REDDIT,NUNL,TWITTER]:
    x[source] = [datetime.datetime.strptime(k,DATEFORMAT) for k in results[source].keys()]
    ySupports[source] = movingAverage(movingAverage([results[source][k][SUPPORTS] for k in results[source].keys()],DAYSCOMBINED),EXTRASMOOTHDAYS)
    xPart = [x[source][i] for i in range(0,len(x[source])) if x[source][i] >= datetime.datetime.strptime(STARTDATE,DATEFORMAT)] 
    ySupportsPart = [ySupports[source][i] for i in range(0,len(x[source])) if x[source][i] >= datetime.datetime.strptime(STARTDATE,DATEFORMAT)] 
    plt.plot_date(xPart,ySupportsPart,fmt="-",label=LABELS[source])
plt.legend(framealpha=0.3,loc="lower right")
plt.ylabel("percentage")
plt.xlabel("day/month")
if TOPIC == FACEMASK:
    plt.title("Support for general public not wearing face masks per medium over time")
elif TOPIC == DISTANCE:
    plt.title("Support for social distancing per medium over time (average over "+str(DAYSCOMBINED)+" days)")
else:
    print("unknown topic!",file=sys.stderr)
    sys.exit(1)    
ax.xaxis.set_major_formatter(mdates.DateFormatter("%-d/%-m"))
plt.savefig(PLOTFILENAME)
plt.show()<jupyter_output><empty_output><jupyter_text>Do not run any further code<jupyter_code>ANNOTATED = "annotated"
ANNOTATEDFILE = "manual-annotation.csv"
DATE = "date"
EENS = "EENS"
ONEENS = "ONEENS"
ANDERS = "ANDERS"
TOTAL = "total"
df = pd.read_csv(ANNOTATEDFILE,index_col=DATE,dtype={EENS:"float",ONEENS:"float",ANDERS:"float"})
totals = []
for i in range(0,len(df)):
    total = df.iloc[i][EENS]+df.iloc[i][ONEENS]+df.iloc[i][ANDERS]
    if df.iloc[i][EENS]+df.iloc[i][ONEENS] > 0:
        df.iloc[i][EENS] = 100*df.iloc[i][EENS]/(total-df.iloc[i][ANDERS])
        df.iloc[i][ONEENS] = 100*df.iloc[i][ONEENS]/(total-df.iloc[i][ANDERS])
        df.iloc[i][ANDERS] = 100*df.iloc[i][ANDERS]/(total-df.iloc[i][ANDERS])
    else: 
        df.iloc[i][ANDERS] = 100
    totals.append(total)
df[TOTAL] = totals
annotatedDict = df.T.to_dict(orient="dict")
annotatedDict = {d:annotatedDict[d] for d in sorted(annotatedDict.keys())}
results[ANNOTATED] = annotatedDict
PLOTFILEANNOTATED = "manual-annotation.png"

x = {}
ySupports = {}
plt.subplots(figsize=(12,6))
font = {"size":12}
matplotlib.rc("font",**font)
ax = plt.subplot(111)
for source in [TWITTER]:
    x[source] = [datetime.datetime.strptime(k,DATEFORMAT) for k in results[source].keys()]
    ySupports[source] = movingAverage([results[source][k][SUPPORTS] for k in results[source].keys()],DAYSCOMBINED)
    xPart = [x[source][i] for i in range(0,len(x[source])) if x[source][i] >= datetime.datetime.strptime(STARTDATE,DATEFORMAT)] 
    ySupportsPart = [ySupports[source][i] for i in range(0,len(x[source])) if x[source][i] >= datetime.datetime.strptime(STARTDATE,DATEFORMAT)] 
    plt.plot_date(xPart,ySupportsPart,fmt="-",label="predicted")
for source in [ANNOTATED]:
    x[source] = [datetime.datetime.strptime(str(k),DATEFORMAT) for k in results[source].keys()]
    ySupports[source] = movingAverage([results[source][k][EENS] for k in results[source].keys()],DAYSCOMBINED)
    xPart = [x[source][i] for i in range(0,len(x[source])) if x[source][i] >= datetime.datetime.strptime(STARTDATE,DATEFORMAT)] 
    ySupportsPart = [ySupports[source][i] for i in range(0,len(x[source])) if x[source][i] >= datetime.datetime.strptime(STARTDATE,DATEFORMAT)] 
    plt.plot_date(xPart,ySupportsPart,fmt="-",label=ANNOTATED)
plt.title("Support for social distancing per medium over time (average over "+str(DAYSCOMBINED)+" days)")
plt.legend()
plt.savefig(PLOTFILEANNOTATED)
plt.show()
annotatedValues = []
twitterValues = []

for i in range(0,len(x[TWITTER])):
    date = x[TWITTER][i]
    if date >= datetime.datetime.strptime("20200310",DATEFORMAT):
        for j in range(0,len(x[ANNOTATED])):
            if x[ANNOTATED][j] == date:
                twitterValues.append(ySupports[TWITTER][i])
                annotatedValues.append(ySupports[ANNOTATED][j])
np.corrcoef(annotatedValues,twitterValues)[1][0]
PLOTFILENAME = "plot.png"

plt.figure()
plt.scatter(annotatedValues,twitterValues)
plt.ylabel("Predicted support percentages")
plt.xlabel("Annotated support percentages")
plt.title("Annotated and predicted support for social distancing")
plt.savefig(PLOTFILENAME)
plt.show()
allResults = results
results = results[TWITTER]

x = [datetime.datetime.strptime(k,DATEFORMAT) for k in results.keys()]
ySupports = movingAverage([results[k][SUPPORTS] for k in results.keys()],DAYSCOMBINED)
yRejects = movingAverage([results[k][REJECTS] for k in results.keys()],DAYSCOMBINED)
yIrrelevant = movingAverage([results[k][IRRELEVANT] for k in results.keys()],DAYSCOMBINED)

keyDates = []
for i in range(0,len(yIrrelevant)):
    if i < len(yIrrelevant)-1 and \
       (yIrrelevant[i] < 100 and yIrrelevant[i+1] >= 100 or yIrrelevant[i] >= 100 and yIrrelevant[i+1] < 100):
        keyDates.append(list(results.keys())[i])
        print(keyDates[-1])
STARTDATE = "20200310" # keyDates[0]
ENDDATE = "20201231"

results = {k:results[k] for k in sorted(results.keys())}
font = {"size":14}
matplotlib.rc("font",**font)
plt.subplots(figsize=(14,7))

ax1 = plt.subplot(111)
plt.plot_date(x,ySupports,fmt="-",label="Supports")
plt.plot_date(x,yRejects,fmt="-",label="Rejects")
plt.plot_date(x,yIrrelevant,fmt="-",label="Other")
plt.plot_date(x,[100 for i in x],fmt="-",label="100%",color="black")
for date in [STARTDATE]:
    plt.plot_date([datetime.datetime.strptime(date,DATEFORMAT)],[100],color="black")
plt.legend()
plt.ylabel("percentage")
plt.xlabel("day/month")
plt.title("stance on the RIVM policy on "+"social")
plt.show()
PLOTFILEOUT = "social-distancing.png"
#PLOTFILEOUT = "mondkapje.png"

plt.subplots(figsize=(14,7))

dates = [d for d in results.keys()]
xPart = [x[i] for i in range(0,len(dates)) if dates[i] >= STARTDATE and dates[i] <= ENDDATE]
ySupportsPart = [ySupports[i] for i in range(0,len(dates)) if dates[i] >= STARTDATE and dates[i] <= ENDDATE] 
yRejectsPart = [yRejects[i] for i in range(0,len(dates)) if dates[i] >= STARTDATE and dates[i] <= ENDDATE]
ax1.xaxis.set_major_formatter(mdates.DateFormatter("%-d/%-m"))

ax2 = plt.subplot(111)
plt.plot_date(xPart,ySupportsPart,fmt="-",label="Supports")
plt.plot_date(xPart,yRejectsPart,fmt="-",label="Rejects")
plt.legend()
plt.ylabel("percentage")
plt.xlabel("day/month")
plt.title("Twitter stance on the RIVM policy on social distancing (average over "+str(DAYSCOMBINED)+" days)")
plt.xticks([datetime.datetime.strptime(d,DATEFORMAT) for d in "20200301 2020401 20200501 20200601 20200701 20200801".split()])
ax2.xaxis.set_major_formatter(mdates.DateFormatter("%-d/%-m"))

plt.savefig(PLOTFILEOUT)
plt.show()
counts = {}
for text in fasttextData:
    label = text.split()[0]
    if label in counts: counts[label] += 1
    else: counts[label] = 1
for label in counts: print(round(counts[label]/len(fasttextData),3),label)<jupyter_output><empty_output><jupyter_text>Annotator comparison<jupyter_code>NBROFTESTDATA = 100

annotators = []
annotatorData = {}
kappaData = [{},{}]
blockedTweetIds = {}
for i in range(0,len(annotations)):
    annotator = annotations.iloc[i][0]
    tweetId = annotations.iloc[i][2]
    tweetLabel = annotations.iloc[i][4]
    if not annotator in annotatorData: 
        annotatorData[annotator] = {}
        annotators.append(annotator)
    annotatorData[annotator][tweetId] = tweetLabel
    if len(annotatorData[annotator]) <= NBROFTESTDATA: blockedTweetIds[tweetId] = True
    if len(annotators) >= 2 and not tweetId in blockedTweetIds and \
       tweetId in annotatorData[annotators[0]] and tweetId in annotatorData[annotators[1]]:
        kappaData[0][tweetId] = annotatorData[annotators[0]][tweetId]
        kappaData[1][tweetId] = annotatorData[annotators[1]][tweetId]
for annotator in annotatorData:
    labelCount = {}
    for tweetId in annotatorData[annotator]:
        label = annotatorData[annotator][tweetId]
        if not label in labelCount: labelCount[label] = 0
        labelCount[label] += 1
    print(len(annotatorData[annotator]))
    for label in labelCount: print(round(labelCount[label]/len(annotatorData[annotator]),3),label)
from sklearn.metrics import cohen_kappa_score

cohen_kappa_score([kappaData[0][tweetId] for tweetId in kappaData[0]],[kappaData[1][tweetId] for tweetId in kappaData[0]])
MAX = 100

total = 0
identical = 0
for tweetId in annotatorData[mainAnnotator]:
    for annotator in annotators:
        if not annotator == mainAnnotator and tweetId in annotatorData[annotator]:
            total += 1
            if total <= MAX:
                if annotatorData[annotator][tweetId] == annotatorData[mainAnnotator][tweetId]: identical += 1
                else: 
                    print(tweetId,annotatorData[mainAnnotator][tweetId],annotatorData[annotator][tweetId])
                    #print(tweets.loc[tweetId][TEXT])
print(total,identical)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13789293409203912, 'fraction_numerical': 0.027125573172752885, 'mean_word_length': 4.567122866369368, 'pattern_counts': {'":': 3, '<': 77, '<?xml version=': 0, '>': 87, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '534283', 'license': 'Apache-2.0', 'n_tokens_mistral': 13208, 'n_tokens_neox': 12249, 'n_words': 2258, 'path': 'fasttext.ipynb', 'repo_name': 'puregome/notebooks', 'size': '36202'}