<jupyter_start><jupyter_text>Sample 3 Analysis===Iterative analysis.<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib
import os
from tqdm import tqdm
import bz2
import sqlite3
import difflib
import gzip
import json
import re
import hashlib
from datetime import datetime
from datetime import timezone
import nltk
import scipy.stats
import para
from itertools import groupby
from collections import Counter
from IPython.display import display, HTML
git_root_dir = !git rev-parse --show-toplevel
git_root_dir = git_root_dir[0]
git_root_dir
raw_data_dir = "/export/scratch2/wiki_data"
derived_data_dir = os.path.join(git_root_dir, "data", "derived")
raw_data_dir, derived_data_dir
stub_history_dir = os.path.join(derived_data_dir, 'stub-history-all-revisions')
stub_history_dir
working_dir = os.path.join(derived_data_dir, 'revision_sample')
os.makedirs(working_dir, exist_ok=True)
working_dir
start_date = datetime.fromisoformat('2014-04-01').replace(tzinfo=timezone.utc)
start_timestamp = int(start_date.timestamp())
end_date = datetime.fromisoformat('2020-01-01').replace(tzinfo=timezone.utc)
end_timestamp = int(end_date.timestamp())
start_timestamp, end_timestamp<jupyter_output><empty_output><jupyter_text>Matplotlib initialization<jupyter_code># Construct colorplot for matrix plotting
# This sets the colormap that will be used for the matrix plots below
def truncate_colormap(cmap, minval=0.0, maxval=1.0, n=100):
    new_cmap = matplotlib.colors.LinearSegmentedColormap.from_list(
        'trunc({n},{a:.2f},{b:.2f})'.format(n=cmap.name, a=minval, b=maxval),
        cmap(np.linspace(minval, maxval, n)))
    return new_cmap
new_cmap = truncate_colormap(plt.cm.Purples, 0, 0.6)<jupyter_output><empty_output><jupyter_text>Load sample 3<jupyter_code># read in the sample dataframe
revision_sample_dir = os.path.join(derived_data_dir, 'revision_sample')
sample3_filepath = os.path.join(revision_sample_dir, 'sample3_all.pkl')
rev_df = pd.read_pickle(sample3_filepath)
len(rev_df)
# read in the ORES scores
# as sample3 is a superset of sample2 and overlaps with sample1, we also load those scores
revision_sample_dir = os.path.join(derived_data_dir, 'revision_sample')
ores_dfs = []
for fname in ['sample1_ores_scores.csv', 'sample2_ores_scores.csv', 'sample3_ores_scores.csv']:
    sample_filepath = os.path.join(revision_sample_dir, fname)
    ores_dfs.append(pd.read_csv(sample_filepath, header=None, names=['rev_id', 'damaging_prob', 'damaging_pred', 'goodfaith_prob', 'goodfaith_pred']))
ores_df = pd.concat(ores_dfs, axis=0)
# remove invalid scores
ores_df = ores_df[(ores_df.damaging_prob != -1)&(ores_df.goodfaith_prob != -1)]
# remove duplicates
ores_df = ores_df.drop_duplicates(subset='rev_id')
len(ores_df)
# most recently available rev
last_rev = ores_df.iloc[-1].rev_id
last_rev
rev_df = pd.merge(rev_df, ores_df, on='rev_id', how='inner')
len(rev_df)
# set the analysis start time to be the beginning of 2018
analysis_start_date = datetime.fromisoformat('2018-01-01')
analysis_start_date = analysis_start_date.replace(tzinfo=timezone.utc)
analysis_start_timestamp = int(analysis_start_date.timestamp())
print(f"Starting analysis from {datetime.utcfromtimestamp(analysis_start_timestamp)}")
analysis_end_timestamp = rev_df[rev_df.rev_id == last_rev].rev_timestamp.iloc[0]
print(f"Using revisions with ORES scores up to {datetime.utcfromtimestamp(analysis_end_timestamp)}")
rev_df = rev_df[(rev_df.rev_timestamp <= analysis_end_timestamp)&(rev_df.rev_timestamp >= analysis_start_timestamp)]
len(rev_df)
# Damaging filter counts
# https://en.wikipedia.org/wiki/Special:ORESModels
rev_counts = np.zeros((2, 3))
for i in [0, 1]:
    for j in [0, 1, 2]:
        ores_bucket_idx = None
        if j == 0:  # likelygood
            ores_bucket_idx = rev_df.damaging_prob <= 0.329
        elif j == 1:  # neither
            ores_bucket_idx = (rev_df.damaging_prob > 0.329)&(rev_df.damaging_prob < 0.919)
        elif j == 2:  # verylikelybad
            ores_bucket_idx = rev_df.damaging_prob >= 0.919
        rev_counts[i,j] = np.sum((rev_df.is_reverted==i)&(ores_bucket_idx))
rev_counts
# revert status X ORES Damaging filter
# names for filters from https://www.mediawiki.org/wiki/ORES/RCFilters
fig, ax = plt.subplots(1, 1, figsize=(6, 4))

rev_pcts = rev_counts.copy()
for j in range(3):
    filter_total = rev_pcts[:,j].sum()
    rev_pcts[:,j] = rev_pcts[:,j] / filter_total * 100
ax.matshow(rev_pcts, cmap=new_cmap)
plt.xlabel("ORES Damaging Filter")
ores_filter_labels = [f"{display_label}\n{int(rev_counts.sum(axis=0)[i]/1000):,}K\n({int(rev_counts.sum(axis=0)[i]) / len(rev_df) * 100:.1f}%)"
               for i, display_label
               in enumerate(['likelygood', 'other', 'verylikelybad'])
              ]
revert_labels = [f"{display_label}\n{int(np.sum(rev_df.is_reverted == i)/1000):,}K\n({np.sum(rev_df.is_reverted == i) / len(rev_df) * 100:.1f}%)"
               for i, display_label
               in enumerate(['Not Reverted', 'Reverted'])
              ]
plt.xticks(ticks=[0, 1, 2], labels=ores_filter_labels)
plt.yticks(ticks=[0, 1], labels=revert_labels)
for i in range(2):  # revert status
    for j in range(3):  # ores filter
        pct = f"{rev_pcts[i, j]:.2f}%"
        plt.text(j, i, pct, verticalalignment='bottom', horizontalalignment='center')
        count = f"{int(rev_counts[i, j]):,}"
        plt.text(j, i, count, verticalalignment='top', horizontalalignment='center')
plt.axvline(0.5, color='black', linewidth=1)
plt.axvline(1.5, color='black', linewidth=1)
plt.show()<jupyter_output><empty_output><jupyter_text>Load in full revision and revert data<jupyter_code>start = datetime.now()
stub_history_reverts_dir = os.path.join(derived_data_dir, 'stub-history-reverts-v2')
rev_ids_filepath = os.path.join(stub_history_reverts_dir, 'rev_ids_sorted.csv')
df = pd.read_csv(rev_ids_filepath,
                 header=None, 
                 names=['page_id', 'rev_id', 'rev_timestamp', 'user_text', 'user_id', 'is_revert_target', 'is_reverted', 'is_reverting']
)
print(f"{datetime.now() - start}")
len(df)
# Read the revert info
stub_history_reverts_dir = os.path.join(derived_data_dir, 'stub-history-reverts')
revert_df_filepath = os.path.join(stub_history_reverts_dir, 'revert_df.pkl')
revert_df = pd.read_pickle(revert_df_filepath)
len(revert_df)
df.head(3)
revert_df.head(3)<jupyter_output><empty_output><jupyter_text>Other preprocessingAdding other columns for easier analysis.<jupyter_code>revert_df['is_multirevert'] = revert_df.reverted_count > 1<jupyter_output><empty_output><jupyter_text>Changeset identification and analysis<jupyter_code>df = df[(df.rev_timestamp >= analysis_start_timestamp)&(df.rev_timestamp <= analysis_end_timestamp)]
len(df)
curr_page_id = -1
prev_user_text = ""

is_in_changeset_list = []
changeset_ids = []
changeset_id_counter = 0
for row in tqdm(df.itertuples(), total=len(df)):
    page_id = row.page_id
    user_text = row.user_text
    if page_id != curr_page_id:
        curr_page_id = page_id
        is_in_changeset_list.append(False)
        changeset_id = changeset_id_counter
        changeset_id_counter += 1
        changeset_ids.append(changeset_id)
    else:
        if prev_user_text == user_text:
            # this is a changeset!
            if is_in_changeset_list[-1]:
                # this is an existing changeset
                changeset_id = changeset_ids[-1]
            else:  # this is a brand new changeset!
                is_in_changeset_list[-1] = True
                changeset_id = changeset_id_counter
                changeset_id_counter += 1
                changeset_ids[-1] = changeset_id
            is_in_changeset_list.append(True)
            changeset_ids.append(changeset_id)
        else: # this is not a changeset!
            is_in_changeset_list.append(False)
            changeset_id = changeset_id_counter
            changeset_id_counter += 1
            changeset_ids.append(changeset_id)
    prev_user_text = user_text
df['is_in_multirev_changeset'] = is_in_changeset_list
df['changeset_id'] = changeset_ids
changeset_id_counter = Counter(df.changeset_id)
df['changeset_size'] = df.changeset_id.map(lambda changeset_id: changeset_id_counter[changeset_id])
# an absurd 49% of revisions are in changesets
np.sum(df.is_in_multirev_changeset) / len(df)
# number of unique changesets
len(set(df.changeset_id)), len(set(df.changeset_id)) / len(df)
# number of unique multirevision changesets
df_subset = df[df.is_in_multirev_changeset]
len(set(df_subset.changeset_id)), len(set(df_subset.changeset_id)) / len(df_subset)
# 22% of changesets have multiple revisions
len(set(df_subset.changeset_id)) / len(set(df.changeset_id))
rev_id_changeset_dict = {row.rev_id: row.changeset_id for row in df.itertuples()}
plt.hist(df.changeset_size, log=True, bins=range(50))
plt.title(f"Histogram of {len(set(df.changeset_id)):,} changeset sizes")
plt.xlabel("Number of revisions in changeset")
plt.ylabel("Changeset count")
plt.show()
# confirming that the number of >1-revision changeset count is the same as computed above
np.sum(np.array([v for v in changeset_id_counter.values()]) > 1)<jupyter_output><empty_output><jupyter_text>Analyze reverts<jupyter_code>analysis_rev_ids = set(rev_df.rev_id)
revert_subset = revert_df[revert_df.reverting_rev_id.isin(analysis_rev_ids)].copy()
len(revert_subset), len(revert_subset) / len(revert_df)
#revert_subset['all_reverted_in_analysis_set'] = revert_subset.reverted_rev_ids.map(
#    lambda reverted_rev_ids: np.all([rev_id in analysis_rev_ids for rev_id in reverted_rev_ids])
#)
#revert_subset = revert_subset[revert_subset.all_reverted_in_analysis_set].copy()
#len(revert_subset), len(revert_subset) / len(revert_df)
revert_subset['reverted_changeset_ids'] = revert_subset.reverted_rev_ids.map(
    lambda reverted_rev_ids: [rev_id_changeset_dict[rev_id] for rev_id in reverted_rev_ids if rev_id in rev_id_changeset_dict]
)
# only a tiny number of reverts have reverted revisions without changeset assignments
num_changeset_ids = revert_subset.reverted_changeset_ids.map(len)
no_changeset_assigned_count = np.sum(num_changeset_ids < revert_subset.reverted_count)
no_changeset_assigned_count, f"{no_changeset_assigned_count / len(revert_subset) * 100:.2f}%"
revert_subset['reverted_unique_changeset_count'] = revert_subset.reverted_changeset_ids.map(lambda reverted_changeset_ids: len(set(reverted_changeset_ids)))
plt.hist(revert_subset.reverted_unique_changeset_count, log=True, bins=range(10))
plt.title("Distribution of reverted changeset counts per revert")
plt.xlabel("Number of reverted changesets")
plt.ylabel("Revert count")
plt.show()
plt.hist(revert_subset.reverted_count - revert_subset.reverted_unique_changeset_count, log=True)
plt.show()
revert_subset['is_changeset_multirevert'] = revert_subset.reverted_unique_changeset_count > 1
pd.crosstab(revert_subset.is_multirevert, revert_subset.is_changeset_multirevert, margins=True)
# 70% of multi-reverts are actually single-changeset reverts
149438 / 213996
np.sum(revert_subset.reverted_unique_changeset_count >= 2) / np.sum(revert_subset.reverted_unique_changeset_count >= 1)
likelygood = rev_df[rev_df.damaging_prob <= 0.329]
len(likelygood)
reverted_likelygood = likelygood[likelygood.is_reverted == 1]
len(reverted_likelygood)
is_self_revert_list = []
for row in tqdm(revert_df.itertuples(), total=len(revert_df)):
    is_self_revert = row.reverting_user_text in row.reverted_user_texts
    is_self_revert_list.append(is_self_revert)
revert_df['is_self_revert'] = is_self_revert_list
reverting_rev_id_self_revert_dict = {row.reverting_rev_id: row.is_self_revert for row in revert_df.itertuples()}
reverted_likelygood['is_self_revert'] = \
    reverted_likelygood.reverting_rev_id.map(lambda reverting_rev_id: reverting_rev_id_self_revert_dict[reverting_rev_id])
np.sum(reverted_likelygood.is_self_revert) / len(reverted_likelygood)
np.sum(reverted_likelygood.is_self_revert), len(reverted_likelygood)
notlikelygood = rev_df[rev_df.damaging_prob > 0.329]
reverted_notlikelygood = notlikelygood[notlikelygood.is_reverted == 1]
reverted_notlikelygood['is_self_revert'] = \
    reverted_notlikelygood.reverting_rev_id.map(lambda reverting_rev_id: reverting_rev_id_self_revert_dict[reverting_rev_id])
np.sum(reverted_notlikelygood.is_self_revert) / len(reverted_notlikelygood)<jupyter_output>/export/scratch2/levon003/bin/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  after removing the cwd from sys.path.<jupyter_text>Non-reverts<jupyter_code>verylikelybad = rev_df[rev_df.damaging_prob >= 0.919]
len(verylikelybad)
nonreverted_verylikelybad = verylikelybad[verylikelybad.is_reverted == 0]
len(nonreverted_verylikelybad)
rev_id_damaging_dict = {row.rev_id: row.damaging_prob for row in rev_df.itertuples()}
curr_page_id = -1
prev_rev_id = -1
next_rev_id_dict = {}
prev_rev_id_dict = {}
for row in tqdm(df.itertuples(), total=len(df)):
    page_id = row.page_id
    if page_id != curr_page_id:
        curr_page_id = page_id
    else:
        if prev_rev_id != -1:
            next_rev_id_dict[prev_rev_id] = row.rev_id
            prev_rev_id_dict[row.rev_id] = prev_rev_id
    prev_rev_id = row.rev_id
# expected pattern: non-reverted verylikelybad are often preceded by another revision in the same changeset
# basically, these are "partial self-reverts"
is_partial_selfrevert_list = []
for row in tqdm(nonreverted_verylikelybad.itertuples(), total=len(nonreverted_verylikelybad)):
    rev_id = row.rev_id
    if rev_id not in rev_id_changeset_dict:
        continue
    changeset_id = rev_id_changeset_dict[rev_id]
    
    # is changeset
    is_partial_selfrevert = False
    while True:
        if rev_id in next_rev_id_dict:
            next_rev_id = next_rev_id_dict[rev_id]
            if next_rev_id in rev_id_changeset_dict and rev_id_changeset_dict[next_rev_id] == changeset_id:
                if next_rev_id in rev_id_damaging_dict and rev_id_damaging_dict[next_rev_id] < 0.919:
                    # following non-damaging edit in the same changeset!
                    is_partial_selfrevert = True
                    break
                else:
                    rev_id = next_rev_id
            else:
                break
        else:
            break
    
    is_partial_selfrevert_list.append(is_partial_selfrevert)
partial_selfrevert_count = np.sum(is_partial_selfrevert_list)
partial_selfrevert_count, partial_selfrevert_count / len(nonreverted_verylikelybad)
nonreverted_verylikelybad['is_partial_selfrevert'] = is_partial_selfrevert_list
nonreverted_verylikelybad = pd.merge(nonreverted_verylikelybad, df[['rev_id', 'changeset_id', 'changeset_size']], on='rev_id', how='left')
# no nas introduced when merging in changeset info
np.sum(nonreverted_verylikelybad.isna())
nonreverted_verylikelybad_subset = nonreverted_verylikelybad[(nonreverted_verylikelybad.is_partial_selfrevert)&(nonreverted_verylikelybad.changeset_size == 2)]
print(len(nonreverted_verylikelybad_subset), len(nonreverted_verylikelybad))
for row in nonreverted_verylikelybad_subset.sample(n=10, random_state=10).itertuples():
    url = f"https://en.wikipedia.org/wiki/?diff={row.rev_id}"
    
    changeset_scores = []
    rev_id = row.rev_id
    changeset_scores.append(rev_id_damaging_dict[rev_id])
    changeset_id = rev_id_changeset_dict[rev_id]
    while True:
        if rev_id in next_rev_id_dict:
            next_rev_id = next_rev_id_dict[rev_id]
            if next_rev_id in rev_id_changeset_dict and rev_id_changeset_dict[next_rev_id] == changeset_id:
                if next_rev_id in rev_id_damaging_dict:
                    changeset_scores.append(rev_id_damaging_dict[next_rev_id])
                else:
                    changeset_scores.append("n/a")
                rev_id = next_rev_id
            else:
                #print("Out of changeset...")
                break
        else:
            #print("No next rev...")
            break
    # now retrieve any previous entries...
    rev_id = row.rev_id
    while True:
        if rev_id in prev_rev_id_dict:
            prev_rev_id = prev_rev_id_dict[rev_id]
            if prev_rev_id in rev_id_changeset_dict and rev_id_changeset_dict[prev_rev_id] == changeset_id:
                if prev_rev_id in rev_id_damaging_dict:
                    changeset_scores.insert(0, f"p({rev_id_damaging_dict[prev_rev_id]:.2f})")
                else:
                    changeset_scores.append("p(n/a)")
                rev_id = prev_rev_id
            else:
                break
        else:
            break
    score_string = ",".join([f"{s:.2f}" if type(s) == float else s for s in changeset_scores])
    print(f"{row.rev_id}\t{url}\t{score_string}")
len(set(nonreverted_verylikelybad[nonreverted_verylikelybad.is_partial_selfrevert].rev_id))
with open("/export/scratch2/levon003/repos/wiki-ores-feedback/data/derived/text_content/revid_reverted_likelygood.txt", 'w') as outfile:
    for rev_id in reverted_likelygood.rev_id:
        if rev_id and rev_id != -1:
            outfile.write(f"{rev_id}\n")
print("Finished.")
nonreverted_verylikelybad.head()
833042571 in set(nonreverted_verylikelybad[nonreverted_verylikelybad.is_partial_selfrevert].rev_id)
# to investigate: why is 830131514 not considered a revert of 830131461?
# my answer from inspecting the diff: a single trailing space was deleted
# thus, this is a "de-facto" revert
# want to automate identifying partial reverts that result only in whitespace differences
df[df.rev_id == 830131461]
def get_db(db_filename):
    db = sqlite3.connect(
            db_filename,
            detect_types=sqlite3.PARSE_DECLTYPES
        )
    db.row_factory = sqlite3.Row
    return db


def get_revision_texts(rev_ids):
    texts = []
    try:
        db_filepath = "/export/scratch2/levon003/repos/wiki-ores-feedback/data/derived/text_content/revisionText.sqlite"
        db = get_db(db_filepath)
        for rev_id in rev_ids:
            rev_id = int(rev_id)
            cursor = db.execute("SELECT rev_id, content, comment, tags FROM revisionText WHERE rev_id = ?", (rev_id,))
            result = cursor.fetchall()
            # unlike the others, there might be multiple comments
            if len(result) > 1:
                # This should never happen if the database is properly constructed...
                print("WARNING: Duplicated rev_id in database, check integrity.")
            if len(result) == 0:
                print(f"Failed to find rev_id {rev_id} in database.")
                continue
            result = result[0]
            d = {
                'rev_id': rev_id,
                'content': result['content'],
                'comment': result['comment'],
                'tags': result['tags'],
            }
            texts.append(d)
    finally:
        db.close()
    return texts
texts = get_revision_texts(nonreverted_verylikelybad.rev_id)
len(texts), len(nonreverted_verylikelybad)
text_df = pd.DataFrame(texts)
text_df.head()
np.sum(text_df.comment == "")
Counter(text_df.tags).most_common(10)
# revision 820214494 has been deleted
set(nonreverted_verylikelybad.rev_id) - set(text_df.rev_id)
# compare the prev and next revisions nonreverted_verylikelybad_subset
sdf = nonreverted_verylikelybad.copy()
start = datetime.now()
rev_ids = np.concatenate((sdf.prev_rev_id, sdf.rev_id, sdf.next_rev_id))
texts = get_revision_texts(rev_ids)
text_df = pd.DataFrame(texts)
sdf = pd.merge(sdf, text_df[['rev_id', 'content']], left_on='prev_rev_id', right_on='rev_id', suffixes=('', '_copy')).rename(columns={
    'content': 'prev_content'
}).drop(columns='rev_id_copy')
sdf = pd.merge(sdf, text_df[['rev_id', 'content']], left_on='next_rev_id', right_on='rev_id', suffixes=('', '_copy')).rename(columns={
    'content': 'next_content'
}).drop(columns='rev_id_copy')
sdf = pd.merge(sdf, text_df[['rev_id', 'content', 'comment']], on='rev_id').rename(columns={
    'content': 'rev_content',
    'comment': 'rev_comment',
})
#sdf = pd.merge(sdf, df[['rev_id', 'changeset_size']], on='rev_id')
print(f"{datetime.now() - start}")
len(sdf)
is_nonwhitespace_revert_list = []
for row in sdf.itertuples():
    prev_content = re.sub(r"\s+", '', row.prev_content)
    next_content = re.sub(r"\s+", '', row.next_content)
    is_nonwhitespace_revert = False
    if prev_content == next_content:
        is_nonwhitespace_revert = True        
    is_nonwhitespace_revert_list.append(is_nonwhitespace_revert)
sdf['is_nonwhitespace_revert'] = is_nonwhitespace_revert_list
np.sum(sdf.is_nonwhitespace_revert)
# nearly 5% meet are partial "non-whitespace" reverts
# meaning that the text is identical but for whitespace, not including carriage returns
np.sum(sdf.is_nonwhitespace_revert) / len(sdf)
np.sum(sdf.changeset_size > 1)
# print a few diffs for quick inspection
sample_list = []
context_chars = 50
for row in sdf[(sdf.changeset_size == 1)&(~sdf.is_nonwhitespace_revert)].sample(n=50, random_state=0).itertuples():
    comment = row.rev_comment        
    url = f"https://en.wikipedia.org/wiki/?diff={row.rev_id}"

    diff_str = ""
    prev_a_start = None
    prev_b_start = None
    prev_size = -1
    l1, l2 = row.prev_content, row.rev_content
    for a_start, b_start, size in difflib.SequenceMatcher(a=l1, b=l2).get_matching_blocks():
        if prev_a_start is not None:
            a_diff = l1[prev_a_start + prev_size:a_start]
            b_diff = l2[prev_b_start + prev_size:b_start]
            if a_diff != b_diff:
                diff_str += f"▛{a_diff}❚{b_diff}▜"
        if size > context_chars * 2 + 5:
            diff_str += l1[a_start:a_start+context_chars] + "<...>" + l1[a_start+size-context_chars:a_start+size]
        else:
            diff_str += l1[a_start:a_start+size]
        prev_a_start = a_start
        prev_b_start = b_start
        prev_size = size

    d = {
        'rev_id': row.rev_id,
        'url': url,
        'comment': row.rev_comment,
        'diff': diff_str
    }
    sample_list.append(d)
sample = pd.DataFrame(sample_list)
sample.head()
len(sample), len(sdf[(sdf.changeset_size == 1)&(~sdf.is_nonwhitespace_revert)])
sample_filepath = "/export/scratch2/levon003/repos/wiki-ores-feedback/data/derived/sample3-annotation/nonreverted_vlb_nonmulti_nonwhitespace_n50.csv"
sample.to_csv(sample_filepath)
print("Finished.")
# print a few diffs for quick inspection
context_chars = 50
for row in sdf.sample(n=50, random_state=0).itertuples():
    if not row.is_nonwhitespace_revert:
        prev_content = row.prev_content.splitlines(keepends=True)
        next_content = row.rev_content.splitlines(keepends=True)
        
        comment = row.rev_comment
        print(f"Comment: '{comment}'")
        
        url = f"https://en.wikipedia.org/wiki/?diff={row.rev_id}"
        print(url)
        
        diff_str = ""
        prev_a_start = None
        prev_b_start = None
        prev_size = -1
        l1, l2 = row.prev_content, row.rev_content
        for a_start, b_start, size in difflib.SequenceMatcher(a=l1, b=l2).get_matching_blocks():
            if prev_a_start is not None:
                a_diff = l1[prev_a_start + prev_size:a_start]
                b_diff = l2[prev_b_start + prev_size:b_start]
                if a_diff != b_diff:
                    diff_str += f"▛{a_diff}❚{b_diff}▜"
            if size > context_chars * 2 + 5:
                diff_str += l1[a_start:a_start+context_chars] + "<...>" + l1[a_start+size-context_chars:a_start+size]
            else:
                diff_str += l1[a_start:a_start+size]
            prev_a_start = a_start
            prev_b_start = b_start
            prev_size = size
        print(diff_str)
        print()
                
        print(f"~~End of diffs in {row.rev_id}~~\n")
nonreverted_verylikelybad_subset = nonreverted_verylikelybad[(nonreverted_verylikelybad.is_partial_selfrevert)&(nonreverted_verylikelybad.changeset_size == 2)]
print(len(nonreverted_verylikelybad_subset))
rev_ids = np.concatenate((nonreverted_verylikelybad_subset.prev_rev_id, nonreverted_verylikelybad_subset.rev_id, nonreverted_verylikelybad_subset.next_rev_id))
texts = get_revision_texts(rev_ids)
text_df = pd.DataFrame(texts)
len(text_df), len(rev_ids)
# compare the prev and next revisions nonreverted_verylikelybad_subset
sdf = nonreverted_verylikelybad_subset.copy()
sdf = pd.merge(sdf, text_df[['rev_id', 'content']], left_on='prev_rev_id', right_on='rev_id', suffixes=('', '_copy')).rename(columns={
    'content': 'prev_content'
}).drop(columns='rev_id_copy')
sdf = pd.merge(sdf, text_df[['rev_id', 'content']], left_on='next_rev_id', right_on='rev_id', suffixes=('', '_copy')).rename(columns={
    'content': 'next_content'
}).drop(columns='rev_id_copy')
is_nonwhitespace_revert_list = []
for row in sdf.itertuples():
    prev_content = re.sub(r"\s+", '', row.prev_content)
    next_content = re.sub(r"\s+", '', row.next_content)
    is_nonwhitespace_revert = False
    if prev_content == next_content:
        is_nonwhitespace_revert = True
    else:
        prev_content = row.prev_content.splitlines(keepends=True)
        next_content = row.next_content.splitlines(keepends=True)
        
        url = f"https://en.wikipedia.org/wiki/?diff={row.rev_id}"
        print(url)
        
        diff_count = 0
        line_count = 0
        prev_diff_line = -1
        for l1, l2 in zip(row.prev_content.splitlines(),row.next_content.splitlines()):
            if l1 != l2:
                # TODO could adjust this to do a look-ahead: 
                # if the next line is also mismatched, assume this is a multi-line change,
                # and use all lines together (with \n's) in the SequenceMatcher
                diff_count += 1
                if diff_count >= 2 and prev_diff_line + 1 == line_count:
                    print("<Changes in adjacent rows, snipping rest of revision diff.>")
                    break                
                diff_str = ""
                prev_a_start = None
                prev_b_start = None
                prev_size = -1
                for a_start, b_start, size in difflib.SequenceMatcher(a=l1, b=l2).get_matching_blocks():
                    #if size == 0:
                    #    continue
                    if prev_a_start is not None:
                        a_diff = l1[prev_a_start + prev_size:a_start]
                        b_diff = l2[prev_b_start + prev_size:b_start]
                        if a_diff != b_diff:
                            diff_str += f"▛{a_diff}❚{b_diff}▜"
                    diff_str += l1[a_start:a_start+size]
                    prev_a_start = a_start
                    prev_b_start = b_start
                    prev_size = size
                print(diff_str)
                print()
                
                prev_diff_line = line_count
            line_count += 1
        print(f"~~End of diffs in {row.rev_id}~~\n")
        
        #diff = difflib.ndiff(prev_content, next_content)
        #print(''.join(diff), end="")

        #table = difflib.HtmlDiff(tabsize=2, wrapcolumn=75).make_table(prev_content, next_content, context=True)
        #display(HTML(table))
    is_nonwhitespace_revert_list.append(is_nonwhitespace_revert)
#sdf['is_nonwhitespace_revert'] = is_nonwhitespace_revert_list
#np.sum(sdf.is_nonwhitespace_revert)

diff_str = ""
prev_a_start = None
prev_b_start = None
prev_size = -1
for a_start, b_start, size in difflib.SequenceMatcher(a=l1, b=l2).get_matching_blocks():
    if size == 0:
        continue
    if prev_a_start is not None:
        a_diff = l1[prev_a_start + prev_size:a_start]
        b_diff = l2[prev_b_start + prev_size:b_start]
        diff_str += f"▛{a_diff}―{b_diff}▜"
    diff_str += l1[a_start:a_start+size]
    prev_a_start = a_start
    prev_b_start = b_start
    prev_size = size
print(diff_str)
l1[145]
np.sum(sdf.is_nonwhitespace_revert) / len(sdf)
set("test\t") ^ set("test ")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1100510710739112, 'fraction_numerical': 0.012342176195204994, 'mean_word_length': 3.8365351629502573, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 52, 'https://': 6, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '592584', 'license': 'MIT', 'n_tokens_mistral': 10255, 'n_tokens_neox': 9878, 'n_words': 1755, 'path': 'notebook/Sample3Analysis.ipynb', 'repo_name': 'levon003/wiki-ores-feedback', 'size': '28196'}