<jupyter_start><jupyter_text>Read the data rows<jupyter_code>rows = []
for input_f in INPUTS:
    with open(os.path.join(CF, input_f)) as f:
        rows += list(csv.DictReader(f))<jupyter_output><empty_output><jupyter_text>Go over the data to count workers' mistakes<jupyter_code>workers = collections.defaultdict(lambda: 0)
inconsistent_workers = collections.defaultdict(lambda: 0)
malicious_workers = collections.defaultdict(lambda: 0)
suspicious_workers = collections.defaultdict(lambda: 0)
log_id_to_snippet_text = {}
for row in rows:
    if DAYS is not None and row['_started_at'].split()[0] not in DAYS:
        continue
    q = row[orig_query[MODE]]
    worker_id = row['_worker_id']
    workers[worker_id] += 1
    if MODE == 'D' and is_inconsistent(row):
        inconsistent_workers[worker_id] += 1
        
    snippet = bs4.BeautifulSoup(row['snippet'], 'lxml')
    snippet_text = snippet.get_text().encode('utf-8')
    log_id_to_snippet_text[row['log_id']] = snippet_text
    if is_malicious(row, MODE, snippet_text):
        malicious_workers[worker_id] += 1
    if is_suspicious(row):
        suspicious_workers[worker_id] += 1<jupyter_output><empty_output><jupyter_text>Normalize<jupyter_code>def normalized(d):
    return collections.defaultdict(lambda: 0, {k: (v / workers[k]) for k, v in d.iteritems() if workers[k] >= MIN_JUDGEMENTS_PER_WORKER})

inconsistent_workers = normalized(inconsistent_workers)
malicious_workers = normalized(malicious_workers)
suspicious_workers = normalized(suspicious_workers)<jupyter_output><empty_output><jupyter_text>Report top bad workers<jupyter_code>def format_xyz_workers(xyz_workers, reverse=True):
    return '\n'.join('worker_id: %s; value: %.2f' % (k, v) for k, v in sorted(
        xyz_workers.iteritems(), key=lambda p: p[1], reverse=reverse)[:10],)

print 'Inconsistent:'
print format_xyz_workers(inconsistent_workers)

print '-' * 80
print 'Malicious:'
print format_xyz_workers(malicious_workers)

print '-' * 80
print 'Both:', set(inconsistent_workers.iterkeys()) & set(malicious_workers.iterkeys())

print '-' * 80
print 'Suspicious:'
print format_xyz_workers(suspicious_workers)<jupyter_output><empty_output><jupyter_text>Decide who is spammer<jupyter_code>labels = collections.defaultdict(lambda: {})
item_scores = collections.defaultdict(lambda: collections.defaultdict(lambda: 0))
potential_bad_abandonments = collections.defaultdict(lambda: {'num': 0, 'denom': 0})

aggregated = {}
num_skipped_labels = 0
skipped_workers = set()
worker_trust = {}
for row in rows:
    worker_id = row['_worker_id']
    log_id = row['log_id']
    if workers[worker_id] < MIN_JUDGEMENTS_PER_WORKER \
            or SKIP_INCONSISTENT_WORKERS and worker_id in inconsistent_workers \
            or malicious_workers[worker_id] > MALICIOUS_WORKER_THRESHOLD \
            or suspicious_workers[worker_id] > SUSPICIOUS_WORKER_THRESHOLD:
        num_skipped_labels += 1
        skipped_workers.add(worker_id)
        continue
    try:
        relevance = int(row[rel_column[MODE]][1:])
    except ValueError:
        relevance = -4
    if relevance == 0:
        potential_bad_abandonments[row[orig_query[MODE]]]['denom'] += 1
        if row.get('no_detailed') == 'D0.1' or 'bad_abandonment' in row['query'].split():
            potential_bad_abandonments[row[orig_query[MODE]]]['num'] += 1
    trust = float(row['_trust'])
    worker_trust[worker_id] = trust
    item_scores[log_id][relevance] += trust if USE_CF_TRUST else 1
    labels[worker_id][log_id] = relevance

    if DUMP_AGGREGATED_SCORES:
        aggregated.setdefault(log_id,
                {'query': row[orig_query[args.mode]], 'url': row['link'],
                 'text': log_id_to_snippet_text[log_id],
                 'r': {}, 'ambiguous': 0, 'bad_abandonment': 0})
        aggregated[log_id]['r'].setdefault(relevance, 0)
        aggregated[log_id]['r'][relevance] += 1
        aggregated[log_id]['ambiguous'] += 'ambiguous' in row['query'].split()
        aggregated[log_id]['bad_abandonment'] += 'bad_abandonment' in row['query'].split()
print '%d workers should be skipped' % len(skipped_workers)
global_spammers_here = set(workers.iterkeys()) & spammers
print '%d global spammers found' % len(global_spammers_here)
print '%d of global spammers are not counted here' % len(global_spammers_here - skipped_workers)<jupyter_output><empty_output><jupyter_text>Compute the [Crowdtruth](http://crowdtruth.org/)-style scores<jupyter_code>clarity_scores = {k: max(v.itervalues()) / sum(v.itervalues()) for k, v in item_scores.iteritems()}

min_score = np.mean(clarity_scores.values()) - np.std(clarity_scores.values())
unclear_items = set(
        (log_id for log_id, score in clarity_scores.iteritems() if score < min_score))

if not USE_CF_TRUST:
    worker_item_scores = {}
    worker_worker_scores = {}
    for worker_id, w_labels in labels.iteritems():
        worker_cosines = []
        for log_id, r in w_labels.iteritems():
            if SKIP_UNCLEAR_ITEMS and log_id in unclear_items:
                continue
            item_score = copy.copy(item_scores[log_id])
            # exclude the current worker:
            item_score[r] -= 1
            # normalize
            denom = sum(item_score.itervalues())
            if denom > 0:
                worker_cosines.append(item_score[r] / denom)
        worker_item_scores[worker_id] = np.mean(worker_cosines)
        # Now, to worker-worker scores.
        worker_worker_cosines = []
        for worker_id2, w_labels2 in labels.iteritems():
            cosine_w_w2 = []
            if worker_id2 == worker_id:
                continue
            for log_id, r in w_labels.iteritems():
                if (SKIP_UNCLEAR_ITEMS and log_id in unclear_items) or log_id not in w_labels2:
                    continue
                cosine_w_w2.append(1.0 if w_labels2[log_id] == r else 0.0)
            if len(cosine_w_w2) > 0:
                worker_worker_cosines.append(np.mean(cosine_w_w2))
        worker_worker_scores[worker_id] = np.mean(worker_worker_cosines)
print 'Lowest worker-item scores:'
print format_xyz_workers(worker_item_scores, reverse=False)

print '-' * 80
print 'Lowest worker-worker scores:'
print format_xyz_workers(worker_worker_scores, reverse=False)<jupyter_output><empty_output><jupyter_text>Plot the worker-worker and worker-item scores<jupyter_code>X = np.array([worker_item_scores.get(w, 0) for w in workers])
Y = np.array([worker_worker_scores.get(w, 0) for w in workers])
colors = np.array([('red' if w in skipped_workers | global_spammers_here else 'green') for w in workers])
fig, ax = plt.subplots()
ax.scatter(X, Y, c=colors)
plt.show()<jupyter_output><empty_output><jupyter_text>Report Agreement Stats<jupyter_code>MIN_W_I_SCORE = 0.0
MIN_W_W_SCORE = 0.0
l_values = []
filtered_workers = set()
w_filtered = len(skipped_workers)
l_filtered = num_skipped_labels
num_labels = num_skipped_labels
for w in workers:
    if w in skipped_workers:
        continue
    num_labels += len(labels[w])
    if (w in global_spammers_here or 
            worker_item_scores[w] < MIN_W_I_SCORE or
            worker_worker_scores[w] < MIN_W_W_SCORE):
        l_filtered += len(labels[w])
        w_filtered += 1
        filtered_workers.add(w)
    else:
        l_values.append(labels[w])

print '%d workers filtered out of %d (%.1f%%)' % (
        w_filtered, len(workers), w_filtered / len(workers) * 100)
print '%d labels filtered out of %d (%.1f%%)' % (
        l_filtered, num_labels, l_filtered / num_labels * 100)
print 'Average Cohen\'s kappa: %f' % cohen_kappa.cohen_kappa(l_values, missing_functor=lambda x: x < 0)
print 'Krippendorf\'s alpha: %f' % krippendorff_alpha.krippendorff_alpha(l_values, missing_functor=lambda x: x < 0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11851088496715187, 'fraction_numerical': 0.009532397269096998, 'mean_word_length': 4.041558441558442, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '612662', 'license': 'Apache-2.0', 'n_tokens_mistral': 2785, 'n_tokens_neox': 2628, 'n_words': 528, 'path': 'data_analysis/Spammers.ipynb', 'repo_name': 'varepsilon/cas-eval', 'size': '7763'}