<jupyter_start><jupyter_text>In persistence API:- **adds**: adds(non-stopwords) + adds(stopwords)- **adds_surv_48h**: adds including stopwords surviving for 48 hours- **adds_stopword_count**: adds(stopwords)**dels** and **reins** are similar<jupyter_code># Import modules.

## For display.
from IPython.display import clear_output, display

## Data Processing
import pandas as pd

## APIs
from wikiwho_wrapper import WikiWho
from external.wikipedia import WikipediaDV, WikipediaAPI

## Managers
from metrics.conflict import ConflictManager
from metrics.token import TokensManager
# Parameters.
page_id = 23507036  # 23507036, 5544751
language = 'en'  # 'en', 'de'
# Persistence API
wikiwho_api = WikiWho(lng=language)
total_actions = wikiwho_api.dv.edit_persistence(page_id)

## Eliminate columns containing 'persistent'
cols_no_persi = total_actions.columns[~total_actions.columns.str.contains('persistent')]
total_actions = total_actions[cols_no_persi]

## Convert some columns' names to what we could understand ##
total_columns_names = ['total', 'total_surv_48h', 'total_stopword_count']

## Add all actions together and merge them into one aggregation dataframe.
total_actions = total_actions.join(pd.DataFrame(
    total_actions.loc[:,'adds':'adds_stopword_count'].values +\
    total_actions.loc[:,'dels':'dels_stopword_count'].values +\
    total_actions.loc[:,'reins':'reins_stopword_count'].values, 
    index=total_actions.index, 
    columns=total_columns_names
))

## Only editors id? No, we still want to know their names. Use WikipediaAPI to achieve ##
## this. More details see:                                                             ##
## https://github.com/gesiscss/wikiwho_demo/blob/master/external/wikipedia.py          ##
wikipediadv_instance = WikipediaDV(WikipediaAPI(lng=language))

# Unique editor's id for getting their names
unique_ids = total_actions['editor_id'].unique()

# Grab editors' names
grabbed_names = wikipediadv_instance.get_editors(unique_ids).rename(columns = {'userid': 'editor_id'})

## Merge the names of editors to the total actions dataframe ##

# Merge on editors ids
total_actions = total_actions.merge(grabbed_names[['editor_id', 'name']], on='editor_id')

# Adjust the column position and fill NaN by 'Unregistered'
total_actions.insert(3, 'editor', total_actions['name'])
total_actions = total_actions.drop(columns=['name'])
total_actions['editor'] = total_actions['editor'].fillna("Unregistered")

## Continue to use our data from A.1; first convert the date to the type datetime ##
total_actions['year_month'] = pd.to_datetime(total_actions['year_month'])

total_actions
# Standard wikiwho api.
all_content = wikiwho_api.dv.all_content(page_id)
revisions = wikiwho_api.dv.rev_ids_of_article(page_id)

## Conflict manager.
conflict_manager = ConflictManager(all_content, revisions, lng=language, include_stopwords=False)
conflict_manager_inc_sw = ConflictManager(all_content, revisions, lng=language, include_stopwords=True)

conflict_manager.calculate()
conflict_manager_inc_sw.calculate()

clear_output()

## Token manager
all_actions = conflict_manager.all_actions
all_actions_inc_sw = conflict_manager_inc_sw.all_actions

token_manager = TokensManager(all_actions, maxwords=100000)
token_manager_inc_sw = TokensManager(all_actions_inc_sw, maxwords=100000)

adds, dels, reins = token_manager.token_survive()
adds_sw, dels_sw, reins_sw = token_manager_inc_sw.token_survive()
# Compare

## Filter date
def filter_date(df, start, end='2020-04-15'):
    df = df.reset_index()
    
    return df[(df['rev_time'] < end) & (df['rev_time'] > start)]

## Actions statistic from persistence api
group_total = total_actions.groupby(pd.Grouper(
                    key='year_month', freq='M'+'S')).sum().reset_index()

## An example for EN, from persistence api.
group_total['year_month'] = group_total['year_month'].astype(str)
print('Use data from Persistence API:')
display(group_total[group_total['year_month'] == '2012-08-01'])
## Standard WikiWho api.
start_date = '2012-07-31'
end_date = '2012-09-01'
### adds action
df_adds = filter_date(adds, start_date, end_date)
df_adds_sw = filter_date(adds_sw, start_date, end_date)

print('Use data from standard WikiWho API:')
print('Adds including stopwords:', df_adds_sw.shape[0])
print('Adds including stopwords survived:', df_adds_sw[df_adds_sw['survive'] == 1].shape[0])
print('Adds excluding stopwords:', df_adds.shape[0])
print('Adds stopwords count:', df_adds_sw.shape[0] - df_adds.shape[0])
### dels action
df_dels = filter_date(dels, start_date, end_date)
df_dels_sw = filter_date(dels_sw, start_date, end_date)

print('Use data from standard WikiWho API:')
print('Dels including stopwords:', df_dels_sw.shape[0])
print('Dels including stopwords survived:', df_dels_sw[df_dels_sw['survive'] == 1].shape[0])
print('Dels excluding stopwords:', df_dels.shape[0])
print('Dels stopwords count:', df_dels_sw.shape[0] - df_dels.shape[0])
### reins action
df_reins = filter_date(reins, start_date, end_date)
df_reins_sw = filter_date(reins_sw, start_date, end_date)

print('Use data from standard WikiWho API:')
print('Reins including stopwords:', df_reins_sw.shape[0])
print('Reins including stopwords survived:', df_reins_sw[df_reins_sw['survive'] == 1].shape[0])
print('Reins excluding stopwords:', df_reins.shape[0])
print('Reins stopwords count:', df_reins_sw.shape[0] - df_reins.shape[0])<jupyter_output>Use data from standard WikiWho API:
Reins including stopwords: 0
Reins including stopwords survived: 0
Reins excluding stopwords: 0
Reins stopwords count: 0
{'fraction_non_alphanumeric': 0.12324829320876751, 'fraction_numerical': 0.01742723679482573, 'mean_word_length': 6.422666666666666, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '278220', 'license': 'MIT', 'n_tokens_mistral': 1944, 'n_tokens_neox': 1834, 'n_words': 413, 'path': 'Decomposing change actions in Persistence API.ipynb', 'repo_name': 'lzl257/gesis_wikiwho', 'size': '5566'}