<jupyter_start><jupyter_text>Labeled Revision Mock===Mocking labeled revisions so that rev-scoring can be applied.See also: CrossTimeSampling2.ipynb<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib
import mwapi
import mwxml
import mwxml.utilities
import mwcli
import mwreverts
import oresapi
import mwparserfromhell
import os
import requests
from tqdm import tqdm
import bz2
import gzip
import json
import re
import hashlib
from datetime import datetime
import nltk
import scipy.stats
import para
from itertools import groupby
from collections import Counter
import time
git_root_dir = !git rev-parse --show-toplevel
git_root_dir = git_root_dir[0]
git_root_dir
labeled_revisions_filepath = os.path.join(git_root_dir, "data/raw/editquality/datasets/enwiki.labeled_revisions.20k_2015.json")
assert os.path.exists(labeled_revisions_filepath)
!head -n 3 {labeled_revisions_filepath}
revisions_features_filepath = os.path.join(git_root_dir, "data/raw/editquality/datasets/enwiki.labeled_revisions.20k_2015.damaging.tsv")
assert os.path.exists(labeled_revisions_filepath)
training_data_rev_ids = []
with open(labeled_revisions_filepath, 'r') as infile:
    for line in infile:
        rev = json.loads(line)
        rev_id = rev['rev_id']
        training_data_rev_ids.append(rev_id)
len(training_data_rev_ids)
# labeled_revisions.20k_2015 text retrieval
# This approach uses mwapi, but it seems not to work for some reason
assert False
session = mwapi.Session("https://en.wikipedia.org/w/api.php", 
                        timeout=5.0,
                        user_agent="LabeledRevisionMock/0.0 (github.com/levon003/wiki-ores-feedback; <EMAIL>) mwapi/0.5.1")
splits = np.array_split(training_data_rev_ids, int(len(training_data_rev_ids) / 50) + 1)
results = []
for rev_id_list in tqdm(splits):
    assert len(rev_id_list) <= 50
    params = {
        'action': 'query',
        'format': 'json',
        'formatversion': '2',
        'revids': "|".join([str(rev_id) for rev_id in rev_id_list]),
        'prop': 'revisions',
        'rvprop': 'ids|timestamp|user|comment|content|tags',
        'rvslots': 'main',
#        'maxlag': 5
    }
    awaiting_result = True
    while awaiting_result:
        try:
            result = session.get(params)
        except requests.exceptions.Timeout as timeout_ex:
            print(timeout_ex)
            print("Timeout waiting for response. Waiting and trying again...")
            time.sleep(5)
        except mwapi.errors.APIError as ex:
            print(ex)
            print("Maxlag at:", datetime.now())
            # https://www.mediawiki.org/wiki/Manual:Maxlag_parameter
            print("Server response exceeded maxlag time; waiting before retry.")
            time.sleep(5)
        results.append(result)
    time.sleep(0.01)
# labeled_revisions.20k_2015 text retrieval
headers = {
    'User-Agent': 'LabeledRevisionMock/0.0 (github.com/levon003/wiki-ores-feedback; <EMAIL>) requests/2.22.0',
    'From': '<EMAIL>'
}
splits = np.array_split(training_data_rev_ids, int(len(training_data_rev_ids) / 50) + 1)
results = []
for rev_id_list in tqdm(splits):
    assert len(rev_id_list) <= 50
    params = {
        'action': 'query',
        'format': 'json',
        'formatversion': '2',
        'revids': "|".join([str(rev_id) for rev_id in rev_id_list]),
        'prop': 'revisions',
        'rvprop': 'ids|timestamp|user|comment|content|tags',
        'rvslots': 'main',
        'maxlag': 5
    }
    awaiting_result = True
    while awaiting_result:
            response = requests.get("https://en.wikipedia.org/w/api.php", params=params, headers=headers)
            if response.ok:
                result = response.json()
                if 'error' in result:
                    print(result['error'])
                    print("Maxlag at:", datetime.now())
                    # https://www.mediawiki.org/wiki/Manual:Maxlag_parameter
                else:
                    results.append(result)
                    break
            else:
                print("Error:", response)
            print("Server response error; waiting before retry.")
            time.sleep(5)
    time.sleep(0.01)  # this is completely unnecessary I think; sequential requests without waiting is fine
# all batches complete?
assert np.all([result['batchcomplete'] for result in results])
results = [result['query'] for result in results]
len(results)
# Save the retrieved data
working_dir = os.path.join(git_root_dir, "data/derived/labeled-revs")
os.makedirs(working_dir, exist_ok=True)
output_filepath = os.path.join(working_dir, 'labeled_revisions.20k_2015.content.ndjson')
with open(output_filepath, 'w') as outfile:
    for result in results:
        outfile.write(json.dumps(result) + "\n")
print("Finished.")
# Load in the labeled revisions content data
labeled_revs_dir = os.path.join(git_root_dir, "data/derived/labeled-revs")
labeled_revs_filepath = os.path.join(labeled_revs_dir, 'labeled_revisions.20k_2015.content.ndjson')
results = []
with open(labeled_revs_filepath, 'r') as infile:
    for line in infile:
        result = json.loads(line)
        results.append(result)
len(results)
results[0].keys()
# 520 total "bad" rev_ids, with bad revisions appearing in 73% of query batches
len(results[0]['badrevids'].keys())
badrevids_count_list = []
badrevids_total = 0
for result in results:
    if 'badrevids' in result:
        badrevids_count = len(result['badrevids'].keys())
        badrevids_total += badrevids_count
        badrevids_count_list.append(badrevids_count)
    else:
        badrevids_count_list.append(0)
badrevids_count_list = np.array(badrevids_count_list)
badrevids_total, np.sum(badrevids_count_list > 0) / len(results)
revision_timestamps = []
for result in results:
    for page in result['pages']:
        for rev in page['revisions']:
            page_id = page['pageid']
            rev_id = rev['revid']
            rev_timestamp_str = rev['timestamp']
            rev_timestamp = int(datetime.strptime(rev_timestamp_str, "%Y-%m-%dT%H:%M:%SZ").timestamp())
            revision_timestamps.append(rev_timestamp)
revision_timestamps = np.array(revision_timestamps)
plt.hist(revision_timestamps, bins=100)

xticks, _ = plt.xticks()
plt.xticks(xticks, 
           [datetime.utcfromtimestamp(xtick).strftime("%Y-%m-%d") for xtick in xticks], 
           rotation='vertical')
plt.show()
days = [int(datetime.utcfromtimestamp(rt).strftime("%Y%m%d")) for rt in revision_timestamps]
mc = Counter(days).most_common()
for day, count in sorted(mc, key=lambda tup: tup[0])[:10]:
    print(f"{day} ({count}) " + "="*count)
print("<snip>")
for day, count in sorted(mc, key=lambda tup: tup[0])[-10:]:
    print(f"{day} ({count}) " + "="*count)
mc[:10]
features_df = pd.read_csv(revisions_features_filepath, sep='\t', header=0)
len(features_df)
rev_list = []
revisions_with_cache_filepath = os.path.join(git_root_dir, "data/raw/editquality/datasets/enwiki.labeled_revisions.w_cache.20k_2015.json")
with open(revisions_with_cache_filepath, 'r') as infile:
    for line in infile:
        rev = json.loads(line)
        rev_list.append({
            'rev_id': rev['rev_id'],
            'damaging': rev['damaging'],
            'goodfaith': rev['goodfaith']
        })
df = pd.DataFrame(rev_list)
len(df)
df.head()
df = pd.concat([df, features_df], axis=1)
df.head()
rev_ids_with_content = set()
for result in results:
    for page in result['pages']:
        for rev in page['revisions']:
            rev_id = rev['revid']
            rev_ids_with_content.add(rev_id)
df['has_content'] = df.rev_id.map(lambda rev_id: rev_id in rev_ids_with_content)
len(rev_ids_with_content)
f"{np.sum(df.has_content)} / {len(df)} ({np.sum(df.has_content) / len(df)*100:.2f}%) training revisions have associated content."<jupyter_output><empty_output><jupyter_text>Construct rev_id sample<jupyter_code>raw_data_dir = "/export/scratch2/wiki_data"
derived_data_dir = os.path.join(git_root_dir, "data", "derived")
# read in the sample dataframe
revision_sample_dir = os.path.join(derived_data_dir, 'revision_sample')
sample2_filepath = os.path.join(revision_sample_dir, 'sample2_1M.pkl')
rev_df = pd.read_pickle(sample2_filepath)
len(rev_df)
rev_df.head()

mock_template = '{"rev_id": %d, "auto_labeled": false, "damaging": false, "goodfaith": true, "autolabel": {}}'

working_dir = os.path.join(git_root_dir, "data/derived/labeled-revs")
mock_rev_filepath = os.path.join(working_dir, "sample2.mock.json")
with open(mock_rev_filepath, 'w') as outfile:
    for rev_id in rev_df.rev_id:
        line = mock_template % rev_id
        outfile.write(line + "\n")
print("Finished.")
!head -n 3 {mock_rev_filepath}
mock_rev_filepath<jupyter_output><empty_output><jupyter_text>From the `revscoring` repo, using the `wiki-revscoring` conda environment, I ran:```cat /export/scratch2/levon003/repos/wiki-ores-feedback/data/derived/labeled-revs/sample2.mock.json | revscoring extract editquality.feature_lists.enwiki.damaging editquality.feature_lists.enwiki.goodfaith --host https://en.wikipedia.org --extractors 32 --verbose > /export/scratch2/levon003/repos/wiki-ores-feedback/data/derived/labeled-revs/sample2.mock.w_cache.jsonrevscoring dump_cache --input /export/scratch2/levon003/repos/wiki-ores-feedback/data/derived/labeled-revs/sample2.mock.w_cache.json --output /export/scratch2/levon003/repos/wiki-ores-feedback/data/derived/labeled-revs/sample2.mock.damaging.tsv --verbose editquality.feature_lists.enwiki.damaging damaging```<jupyter_code># copied from bash commands above
mock_features_filepath = "/export/scratch2/levon003/repos/wiki-ores-feedback/data/derived/labeled-revs/sample1.mock.damaging.tsv"
mock_features_df = pd.read_csv(mock_features_filepath, sep='\t', header=0)
len(mock_features_df)
features_df.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12705667276051189, 'fraction_numerical': 0.015234613040828763, 'mean_word_length': 3.89170392449081, 'pattern_counts': {'":': 5, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 5, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '50981', 'license': 'MIT', 'n_tokens_mistral': 3481, 'n_tokens_neox': 3308, 'n_words': 651, 'path': 'notebook/LabeledRevisionMock.ipynb', 'repo_name': 'levon003/wiki-ores-feedback', 'size': '9873'}