<jupyter_start><jupyter_text>Select data for annotation<jupyter_code>import os
import pandas as pd
import random
import re
from IPython.display import clear_output
BASEDIR = "/home/erikt/projects/puregome/data/"
DISTANCE = "distance"
FACEMASK = "mondkapje"
IDSTR = "id_str"
NUNL = "nunl"
REDDIT = "reddit"
TEXT = "text"
TWITTER = "twitter"
USER = "user"
def squeal(text=None):
    clear_output(wait=True)
    if not text is None: print(text)
TOPIC = DISTANCE
DATASOURCE = TWITTER
DATADIR = BASEDIR+DATASOURCE+"/text/"

if TOPIC == DISTANCE:
    query = "1[.,]5[ -]*m|afstand.*hou|hou.*afstand|anderhalve[ -]*meter"
elif TOPIC == FACEMASK:
    query = TOPIC
else:
    print(f"unknown topic! ({TOPIC})",file=sys.stderr)
    sys.exit(1)
FILEPATTERN = "202007"

inFileNames = sorted(os.listdir(DATADIR))

matches = []
for inFileName in inFileNames:
    if re.search(FILEPATTERN,inFileName):
        squeal(inFileName)
        try:
            fileData = pd.read_csv(DATADIR+inFileName).drop_duplicates()
            for i in range(0,len(fileData)):
                if re.search(query,fileData.iloc[i][TEXT],flags=re.IGNORECASE):
                    matches.append(fileData.iloc[i])
        except:
            pass
len(matches)
ANNOTATIONFILEALL = f"{TOPIC}-{DATASOURCE}-all.csv"
ANNOTATIONFILESELECTED = f"{TOPIC}-{DATASOURCE}-selected.csv"

pd.DataFrame(matches).to_csv(ANNOTATIONFILEALL,index=False)
PERCENTAGE = 2

selected = []
targetSize = round(len(matches)*PERCENTAGE/100)
while len(selected) < targetSize and len(matches) > 0:
    index = int(random.random()*len(matches))
    selected.append(matches[index])
    matches[index] = matches[-1]
    del(matches[-1])
    
pd.DataFrame(selected).to_csv(ANNOTATIONFILESELECTED,index=False)
len(selected)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13804713804713806, 'fraction_numerical': 0.009539842873176206, 'mean_word_length': 4.29080118694362, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '530788', 'license': 'Apache-2.0', 'n_tokens_mistral': 668, 'n_tokens_neox': 646, 'n_words': 116, 'path': 'annotation-select.ipynb', 'repo_name': 'puregome/notebooks', 'size': '1782'}