<jupyter_start><jupyter_text>Modules Imported<jupyter_code>## Modules Imported ##

# Display
from IPython.display import display, Markdown as md, clear_output
from datetime import date
import urllib

# APIs
from wikiwho_wrapper import WikiWho
from external.wikipedia import WikipediaDV, WikipediaAPI
from external.wikimedia import WikiMediaDV, WikiMediaAPI
from external.xtools import XtoolsAPI, XtoolsDV

# Data Processing
import pickle
import pandas as pd
import numpy as np

# Visualization tools
import qgrid
import matplotlib.pyplot as plt
import plotly
from plotly.subplots import make_subplots

# Page views timeline
from visualization.views_listener import ViewsListener

# Change actions timeline
from visualization.actions_listener import ActionsListener

# Actions per editor visualization
from visualization.editors_listener import EditorsListener
from visualization.editors_listener import RankedEditorsListener

# Conflicts visualization
from visualization.conflicts_listener import ConflictsListener, ConflictsActionListener, ConflictsEditorListener
from visualization.calculator_listener import ConflictCalculatorListener

# Word cloud visualization
from visualization.wordcloud_listener import WCListener, WCActionsListener
from visualization.wordclouder import WordClouder

# Tokens per revision visualization
from visualization.tokens_listener import TokensListener, TokensOwnedListener

# Wikipedia talk pages visualization
from visualization.talks_listener import TalksListener
from visualization.topics_listener import TopicsListener

# Tokens ownership visualization
from visualization.owned_listener import OwnedListener

# Metrics management
from metrics.conflict import ConflictManager
from metrics.token import TokensManager

# For language selection
from utils.lngselection import abbreviation, lng_listener<jupyter_output><empty_output><jupyter_text>You can continue with the article selected in the previous notebook (skip the search button below) or change it  (e.g., if you would like to compare two or more articles):<jupyter_code># the method that listens to the click event
def on_button_clicked(b):
    global the_page, the_talk_page
    
    # use the out widget so the output is overwritten when two or more
    # searches are performed
    with out0:
        try:
            # query wikipedia
            wikipedia_dv = WikipediaDV(WikipediaAPI(lng=abbreviation(languageSelection.value)))
            search_result = wikipedia_dv.search_page(searchTerm.value)
            the_page = wikipedia_dv.get_page(search_result)
            the_talk_page = wikipedia_dv.get_page("Talk:"+search_result)
            %store the_page
            clear_output()
            display(md(f"The page that was found: **{the_page['title']}**"))
            display(md(f"Page id: **{the_page['page_id']}**"))

        except:
            clear_output()
            display(md(f'The page title *"{searchTerm.value}"* was not found'))

# Load the selection from NB1
with open('utils/language.json', 'r') as file:
    json_selection = json.load(file)
    lng_nb1 = json_selection['lng']
    title_nb1 = json_selection['search_term']

# Language selection.
languageSelection = Dropdown(options=['English', 'Deutsch', 'Español', 'Türkçe', 'Euskara'], value=lng_nb1, description='Language:')            

# by default display the last search
try:
    searchTerm = widgets.Text(the_page['title'], description='Page title:')
except:
    searchTerm = widgets.Text(title_nb1, description='Page title:')

out_main = Output()
display(out_main)
with out_main:
    second_select = widgets.interactive(lng_listener, lng=languageSelection, search_term=searchTerm)
    display(second_select.children[0])
    display(second_select.children[1])

    # create and display the button    
    button = widgets.Button(description="Search")
    display(button)

# the output widget is used to remove the output after the search field
out0 = Output()
display(out0)

# set the event
button.on_click(on_button_clicked)

# trigger the event with the default value
on_button_clicked(button)<jupyter_output><empty_output><jupyter_text>---**Click on "Refresh All Variables" to reload all following cells with the selected article. (may take a some time)** Alternatively, reload each module individually via their respective buttons.<jupyter_code>def restart_and_run():
    display(HTML(
        '''
            <script>
                code_show = false;
                function restart_run_all(){
                    IPython.notebook.kernel.restart();
                    setTimeout(function(){
                        IPython.notebook.execute_all_cells();
                    })
                }
            </script>
            <button onclick="restart_run_all()">Refresh All Variables</button>
        '''
    ))
restart_and_run()<jupyter_output><empty_output><jupyter_text>--- Troubleshooting:- Allow some time for the notebook to run fully before interacting with the interface controls. For articles with a long revision history, this could take minutes and the interaction with the controls will be slow.- Very large articles ("<NAME>", "World War 2") may lead to memory overload, depending on which Binder platform you are using. You could get more memory by shutting down some previous running notebooks.- **All cells should run automatically** when you open the notebook and you should not see any code. If that is not the case, please just reload the tab in your browser. - After chosing a new article, please rerun the cells/modules you want to use. - You should not see any code when you run it for the first time. If so, please let us know by posting an issue in our Github repository: https://github.com/gesiscss/IWAAN/issues. --- A. Change actions and their impact**The dynamics of individual changes** made to every word (and each special character) in the article are extracted and collected by the WikiWho service, and we access them through the [WikiWho API](https://www.wikiwho.net/en/api/v1.0.0-beta/). Below, you will find several metrics for such change dynamics such as unsucessful changes (i.e. changes reverted by other editors) and conflict (defined below).         More information about WikiWho's methodThe [WikiWho API](https://www.wikiwho.net/en/api/v1.0.0-beta/) accurately tracks the changes to every token (words or special characters) for live Wikipedia articles. It distinguishes every token in the document even when the string appears several times. For example, "and" at the beginning of an article is a different token than "and" at the end of the article. See also [this figure](https://www.wikiwho.net/technical_details).That means that *not only edits* (== whole revisions) are counted, which can contain changes of many different tokens, but *every single action to every single token* is recorded.        The two most basic actions that can perfomed per token are **insertions** and **deletions**.     (Side note: A character change in a word, e.g. "dog" -> "dogs", is modeled as deletion of "dog" and the insertion of "dogs", two separate tokens.)        An **insertion** is also considered a **re-insertion** if the insertion has occured before; the only insertion of a token that is not a re-insertion is the first one (original addition of the token, or **add**).Formally, the token history can be represented by a time-ordered sequence of actions *(a0, ..., an)*; note that *a0+2i* is always an insertion and *a1+2i* is always a deletion for i ∈ ℕ.   A.1 Actions over time - When did certain change actions happen? Let's start with the simplest form of information that we can directly access from the WikiWho API, i.e. the actions that have been perfomed on the tokens of the articles. The following graph displays the number of actions (y-axis) that occured through time (x-axis). You can interact with this graph by:1. choosing the *date range* and *granularity* (yearly, monthly) of the timeline (X-axis), and 2. selecting any of the *change action types* in the black, red, blue and green traces.         Explanation of the change action types   - **adds**: number of first-time insertions / additions- **adds_surv_48h**: number of adds that survived at least 48 hours (after 48h, the risk of being removed is negligible on avg.)- **adds_stopword_count**:  number of insertions that were only [stop words](https://github.com/gesiscss/wikiwho_demo_new/blob/master/data/stopword_list.txt)- **dels**: number of deletions- **dels_surv_48h**: number of deletions that were not resinserted in the next 48 hours - **dels_stopword_count**: number of deletions that were only [stop words](https://github.com/gesiscss/wikiwho_demo_new/blob/master/data/stopword_list.txt)- **reins**: number of reinsertions- **reins_surv_48h**: number of reinsertions that survived at least 48 hours- **reins_stopword_count**: number of reinsertions that were only [stop words](https://github.com/gesiscss/wikiwho_demo_new/blob/master/data/stopword_list.txt)        ExampleFor instance, if you see 10 "adds" in a month, but only 4 "adds_surv_48h", then 10 completely new tokens/words have been added to the article, but only 4 of them stayed in the article for more than 2 days, which usually means the other 6 are gone for good. If "dels" are performed and don't survive, that means that these deletions have been undone, i.e., the deleted tokens have been reinserted (you should see at least the same amount of reinsertions happen). In other words, these are measurements of the longevity and stability of edit actions done to the article.        How is conflict measured?Our measurement of ***conflict*** for single tokens is taken from [Flöck et al.](https://arxiv.org/abs/1703.08244):* (1)  The main idea is to count how often a token (after being created (added) the first time) has been deleted, re-inserted, re-deleted, re-inserted, and so on; which would often happen in case two editors disagree on the token's justification to be in the text. * (2) Only the **re-**deletions and **re-**insertions are counted, since up to the first delete it could be a simple correction that didn't trigger a reponse - this wouldn't indicate conflict. * (3) The **"re-"** actions are only counted if they alternate between different editors and don't come from the same editor twice or more in a row - as the latter would simply indicate self-corrections. * (4) In a last step, each re-insertion/re-deletion interaction gets a higher weight the faster it occurs (see [Flöck et el.](https://arxiv.org/abs/1703.08244) for the exact formula).The total conflict of a page is the sum of all the conflict scores of all actions with conflict (or conflict actions).This total conflict can be normalized (between 0 and 1) if the sum is divided by the number of actions that could have been potentially counted as conflict, i.e. a **re-**deletion or **re-**insertion (called *elegible actions*).The conflict scores in all notebooks are calculated based on the tokens excluding stopwords.        Vandalism detection with ORES, "Goodfaith" and "Damage" scoresTypically, a lot of the adds, deletes and reinserts are generated by vandalism and vandalism fighting. This obscures the "real" work on the article in the graph below, particularly as these frequently involve large chunks of text being deleted and reinserted. We thus filter out such actions that come from (i) revisions that are probably vandalism **and** (ii) such that repair them. Otherwise, say,  500 tokens reinserted after a not-shown 500 vandalistic deletes would look pretty odd.             This happens via filtering the revisions based on thresholded values for "Goodfaith" (well-intened edit yes/no) and "Damage" (actual quality decrease for the article) scores retrieved live from the [ORES API](https://ores.wikimedia.org/). We found that the threshold-combination of lower than 0.6 damage and higher than 0.6 goodfaith scores work quite well for most articles, but you are encouraged to check for yourself if the vandalism filtering works as intendend or if you have to alter these parameters.         For one or a range of vandalistic revisions, we consider the immediately subsequent one (with non-vandalistic scores) to be the "fixer" and filter it out as well. If you set the thresholds to zero, the filters are deactivated.<jupyter_code>import operator
from external.ores import ORESAPI, ORESDV

def get_ores_scores(rev_list):
    # Get data from ORES
    oapi = ORESAPI()
    ores = ORESDV(oapi)
    max_requests = 50
    rev_list = list(map(str, rev_list))
    res = ores.get_goodfaith_damage(rev_list[0:max_requests])
    i = 1
    while len(res) < len(rev_list): 
        #print(len(res), len(rev_list))
        tmp = ores.get_goodfaith_damage(rev_list[max_requests*i:max_requests*i+max_requests])
        res = res.append(tmp)
        i = i+1
        
    return res

# create and display the button
button2 = widgets.Button(description="Load Actions", layout=Layout(width='160px'))

display(button2)

def on_click_details(b):
    global agg_actions
    global editors
    global action_types
    global sources    
    global actions_inc_stop
    global revisions
    global con_manager_all
    global ores_scores
    global listener
    global all_content
    global wikiwho
    global total_actions
    global token_sources
    with out4:
        clear_output()
       
        # Preparetory works
        wikiwho = WikiWho(lng=abbreviation(languageSelection.value))
        display(md("Downloading all_content from the WikiWhoApi..."))
        all_content = wikiwho.dv.all_content(the_page['page_id'])
        display(md("Downloading revisions from the WikiWhoApi..."))
        revisions = wikiwho.dv.rev_ids_of_article(the_page['page_id'])
        clear_output()
        
        con_manager_all = ConflictManager(all_content, 
                                           revisions.copy(), 
                                           lng=abbreviation(languageSelection.value), 
                                           include_stopwords=True)

        con_manager_all.calculate()
        clear_output()
        
        sources = con_manager_all.source
        sources["wiki_dv"] = WikipediaDV(WikipediaAPI(lng=abbreviation(languageSelection.value)))
        
        listener = ActionsListener(sources=sources, lng=abbreviation(languageSelection.value))
        listener.get_main()
        
        # Filter spam with ORES
        tmp = revisions['rev_id'].to_list()
        ores_scores = get_ores_scores(tmp)
        listener.ores_scores= ores_scores
        
        #not_spam = filter_vandalism_ores(ores_scores, goodfaith_cmp='>', goodfaith_threshold=0.9, damaging_cmp='<=', damaging_threshold=0.09)
        #listener.df = listener.df[~listener.df['rev_id'].isin(not_spam)]
        
        clear_output()
        agg_actions = listener.df.copy()
        actions_inc_stop = listener.tokens_group_all.copy()
        editors = listener.editors.copy()
       
        # Merge the names of the editors to the aggregate actions dataframe
        display(md(f"***Page: {the_page['title']} ({abbreviation(languageSelection.value).upper()}) with {agg_actions['revisions'].sum()} revisions***"))

        # Visualization
        action_types = (agg_actions.columns[5:14]).values.tolist()
        datepicker_from = widgets.DatePicker(description='Date starts', value=agg_actions['rev_time'].iloc[0],
                                             style={'description_width': '40%'}, layout=Layout(width="29%"))
        datepicker_to = widgets.DatePicker(description='Date ends', value=agg_actions['rev_time'].iloc[-1],
                                           style={'description_width': '43%'}, layout=Layout(width="31%"))
        editor_all = fixed('All')
        frequency = Dropdown(options=['Yearly', 'Monthly', 'Weekly', 'Daily'], value='Monthly', description="Granularity",
                             style={'description_width': '43%'}, layout=Layout(width="32%"))
        black_actions = Dropdown(options=action_types, value='adds', description="All Actions(black)", style={'description_width': '40%'})
        red_actions = Dropdown(options= ['None'] + action_types, value="adds_surv_48h", description="All Actions(red)", style={'description_width': '40%'})
        green = Dropdown(options= ['None'] + action_types, value='None', description="All Actions(green)", style={'description_width': '40%'})
        blue = Dropdown(options= ['None'] + action_types, value='None', description="All Actions(blue)", style={'description_width': '40%'})
        
        metrics = ['Absolute Conflict Score', 'Norm. Conflict Score']
        black_conflict = Dropdown(options=metrics, value='Absolute Conflict Score', description="Conflict(black)", style={'description_width': '40%'})
        red_conflict = Dropdown(options= ['None'] + metrics, value='Norm. Conflict Score', description="Conflict(red)", style={'description_width': '40%'})
       
        damage_t = widgets.FloatText(description='Damage Threshold', style={'description_width': '40%'}, value=0.6)
        goodwill_t = widgets.FloatText(description='Goodfaith Threshold', style={'description_width': '40%'}, value=0.6)
        damage_c =  Dropdown(options=['<','>','='], value='<', description="Damage Comparator", style={'description_width': '40%'})
        goodwill_c =  Dropdown(options=['<','>','='], value='>', description="Goodfaith Comparator", style={'description_width': '40%'})
        
        test_fig = make_subplots(rows=2, cols=1, start_cell="bottom-left", shared_xaxes=True, vertical_spacing=0.05)
        
        # All actions
        actions_out = interactive_output(listener.listen, 
            {"_range1": datepicker_from,
            "_range2": datepicker_to,
            "editor": fixed('All'),
            "granularity": frequency,
            "black": black_actions,
            "red": red_actions,
            "green": green,
            "blue": blue,
            "black_conflict": black_conflict,
            "red_conflict": red_conflict,
            "damage_t": damage_t,
            "goodwill_t": goodwill_t,
            "damage_c": damage_c,
            "goodwill_c": goodwill_c})
        display(HBox([datepicker_from, datepicker_to, frequency]))
        display(HBox([black_actions, red_actions, green, blue, black_conflict, red_conflict],
                    layout=Layout(flex_flow='wrap')))
        display(HBox([damage_c, damage_t]))
        display(HBox([goodwill_c, goodwill_t]))
        display(actions_out)
        
        red_actions.value = "dels"
        token_sources = sources.copy()
        total_actions = agg_actions.copy()
    
out4 = Output()
display(out4)

# set the event
button2.on_click(on_click_details)

# trigger the event with the default value
on_click_details(button2)<jupyter_output><empty_output><jupyter_text>Try yourself! This is what will happen when you click 'Load Actions' button:<jupyter_code>%%script false --no-raise-error

## IMPORTANT NOTE: COMMENT THE ABOVE LINE TO EXECUTE THE CELL ###

### --------------------------------------------------------------------------- ###
### TRY YOURSELF! THIS IS WHAT WILL HAPPEN WHEN YOU CLICK 'Load Actions' BUTTON ###
### --------------------------------------------------------------------------- ###

## Create a WikiWho instance, which is an API; and use ConflictManager to get         ##
## actions for given page id; more details see:                                       ##
## https://github.com/gesiscss/wikiwho_wrapper/blob/master/wikiwho_wrapper/api.py     ##
## https://github.com/gesiscss/wikiwho_wrapper/blob/master/wikiwho_wrapper/wikiwho.py ##
## https://github.com/gesiscss/wikiwho_demo_new/blob/master/metrics/conflict.py       ##
# wikiwho_api = WikiWho(lng=abbreviation(languageSelection.value))
# print("Downloading all_content from the WikiWhoApi...")
# all_content = wikiwho_api.dv.all_content(the_page['page_id'])
# print("Downloading revisions from the WikiWhoApi...")
# all_revisions = wikiwho_api.dv.rev_ids_of_article(the_page['page_id'])
# clear_output()

# conflict_manager = ConflictManager(all_content.copy(),
#                                all_revisions.copy(), 
#                                lng=abbreviation(languageSelection.value), 
#                                include_stopwords=True)

# conflict_manager.calculate()
# clear_output()

# Store the original data into variable "token_sources", it includes
# "tokens_all" / "tokens" (all actions of an article, including/excluding stopwords)
# "elegibles_all" / "elegibles" (elegible actions of an article, including/excluding stopwords)
# "conflicts_all" / "conflicts" (conflict actions of an article, including/excluding stopwords)
# "wiki_dv" (a WikipediaDV instance for retrieving editors' names given their IDs)
# token_sources = conflict_manager.source
# token_sources["wiki_dv"] = WikipediaDV(WikipediaAPI(lng=abbreviation(languageSelection.value)))
token_sources = sources.copy()
all_revisions = revisions.copy()

## Create an ActionsListener instance to get aggregated actions. More details see:            ##
## https://github.com/gesiscss/wikiwho_demo_new/blob/master/visualization/actions_listener.py ##
actions_listener = ActionsListener(sources=token_sources, lng=abbreviation(languageSelection.value))
actions_listener.get_main()
clear_output()

# Aggregation data for actions.
total_actions = agg_actions

# adds/dels/reins actions (including stopwords)
actions_inc = actions_listener.tokens_group_all.copy()

# Editors' names.
grabbed_names = actions_listener.editors.copy()

# Filter spam with ORES
tmp = revisions['rev_id'].to_list()
ores_scores = get_ores_scores(tmp)
actions_listener.ores_scores= ores_scores
        
## Visualization, core visual code lies in ActionsListener, then the interact function     ##
## makes it interactive, mode details see:                                                 ##
## https://github.com/gesiscss/wikiwho_demo/blob/master/visualization/actions_listener.py  ##

# This is the page you used
print('The page that is being used:', the_page['title'], f'({abbreviation(languageSelection.value).upper()})')

# Which types do those actions have?
actions_types = (total_actions.columns[5:14]).values.tolist()

# Visualization: try yourself by coding!
begin_date = date(2001, 7, 16)
end_date = date(2020, 12, 5)
editor_name = 'All'
frequency = 'Monthly' # "Yearly", "Monthly", "Weekly", "Daily"
black_line = 'adds' # Choose one from 'actions_types'. Note that 'black_line doesn't have 'None' option but others have.
red_line = 'dels'
blue_line = 'None'
green_line = 'None'
black_conflict = "Absolute Conflict Score"  # 'Conflict Score', 'Absolute Conflict Score', 'Total Elegible Actions'
red_conflict = "None"
damage_t = 0.6
goodwill_t = 0.6
damage_c = "<"
goodwill_c = ">"

actions_listener.listen(
    _range1 = begin_date,
    _range2 = end_date,
    editor = editor_name,
    granularity = frequency,
    black = black_line,
    red = red_line,
    blue = blue_line,
    green = green_line,
    black_conflict = black_conflict,
    red_conflict = red_conflict,
    damage_t = damage_t,
    goodwill_t = goodwill_t,
    damage_c = damage_c,
    goodwill_c = goodwill_c
)<jupyter_output><empty_output><jupyter_text>A.2 Impactful editors - Who has productively contributed?The following elements present the editors who have contributed and whose changes have not been removed for 48 hours, which it is an heuristic to exclude low-quality contributions from being shown. Given time interval and granularity, the first chart ranks all editors by all their edits that have survived for 48 hours and displays top 20/10/5 editors. The second graph then shows the evolution of these actions along the time axis for each editor.<jupyter_code># create and display the button
button_rank = widgets.Button(description="Get Editors Rank", layout=Layout(width='160px'))

display(button_rank)

def on_click_rank(b):
    with out_rank:
        clear_output()
        display(md(f"***Page: {the_page['title']} ({abbreviation(languageSelection.value).upper()})***"))
        
        ranked_listener = RankedEditorsListener(agg_actions)

        datepicker_from = widgets.DatePicker(description='Date starts', value=agg_actions['rev_time'].iloc[0], layout=Layout(width='25%'))
        datepicker_to = widgets.DatePicker(description='Date ends', value=agg_actions['rev_time'].iloc[-1], layout=Layout(width='25%'))
        frequency = Dropdown(description='Granularity', options=['Monthly', 'Weekly', 'Daily', 'Timestamp (Revision)'], value='Monthly')
        top = widgets.RadioButtons(options=[5, 10, 20], value=20, description="Top Editors", layout={'width': '50%'})

        interact(ranked_listener.listen,
                _range1=datepicker_from,
                _range2=datepicker_to,
                granularity=frequency,
                top=top)
        
out_rank = Output()
display(out_rank)

# set the event
button_rank.on_click(on_click_rank)

# trigger the event with the default value
on_click_rank(button_rank)<jupyter_output><empty_output><jupyter_text>Try yourself! This is what will happen when you click 'Get Editors Rank' button:<jupyter_code>%%script false --no-raise-error

### IMPORTANT NOTE: COMMENT THE ABOVE LINE TO EXECUTE THE CELL ###

### ------------------------------------------------------------------------------- ###
### TRY YOURSELF! THIS IS WHAT WILL HAPPEN WHEN YOU CLICK 'Get Editors Info' BUTTON ###
### ------------------------------------------------------------------------------- ###

## Create an RankedEditorsListener instance, more details can be found in                     ##
## https://github.com/gesiscss/wikiwho_demo_new/blob/master/visualization/editors_listener.py ##

rank_listener = RankedEditorsListener(total_actions)

## Visualization, core visual code lies in RankedEditorsListener, then the interact function  ##
## makes it interactive.                                                                      ##

# This is the page you used
print('The page that is being used:', the_page['title'], f'({abbreviation(languageSelection.value).upper()})')

# Visualization: parameters
begin_date = date(2007, 3, 1)
end_date = date(2020, 5, 1)
frequency = 'Monthly' # "Monthly", "Weekly", "Daily"
top_editors = 20 # 20, 10, 5

# Visualization: chart and graph
rank_listener.listen(_range1=begin_date,
                    _range2=end_date,
                    granularity=frequency,
                    top=top_editors)<jupyter_output><empty_output><jupyter_text>A.3 Token ownership - Which editors were original authors of how much text per time frame?The graph below displays the number of tokens "owned" by the top-15 editors for the given time period. Token ownerships are counted as original additions that have not been deleted by the end of a timeframe. The number of total tokens existing on a page can be considered as the top line of a graph. The granularity (daily, weekly or monthly) controls if the latest revision would be considered within a time window. For example, for daily period the latest revision during each day is considered. You can remove editors by clicking on their name in the legend and keep only a specific editor by double-clicking.<jupyter_code># # create and display the button
button_a21 = widgets.Button(description="Show token owners")
display(button_a21)


def on_click_owned_tokens(b):
    with out21:
        clear_output()
        display(md(f"***Page: {the_page['title']} ({abbreviation(languageSelection.value).upper()})***"))
        listener = TokensOwnedListener(agg_actions, sources, lng=abbreviation(languageSelection.value))
    
        # visualization
        #display(datepicker_from, datepicker_to, timeframe, stopwords_a23)
        datepicker_from = widgets.DatePicker(description='Date starts', value=agg_actions['rev_time'].iloc[0], layout=Layout(width="25%"))
        datepicker_to = widgets.DatePicker(description='Date ends', value=agg_actions['rev_time'].iloc[-1], layout=Layout(width="25%"))
        timeframe = widgets.RadioButtons(options=['Monthly', 'Weekly', "Daily"], value='Monthly', description='Granularity:', layout={'width': '50%'})
        stopwords_a23 = widgets.RadioButtons(options=['Not included', 'Included'], value='Not included', description='Stopwords', layout={'width': '50%'})
        interact(listener.listen, _range1=datepicker_from,
                 _range2=datepicker_to,
                 granularity=timeframe,
                 stopwords = stopwords_a23)
        
out21 = Output()
display(out21)

# # set the event
button_a21.on_click(on_click_owned_tokens)

# # trigger the event with the default value
on_click_owned_tokens(button_a21)<jupyter_output><empty_output><jupyter_text>Try yourself! This is what will happen when you click 'Load Actions' button:<jupyter_code>%%script false --no-raise-error

### IMPORTANT NOTE: COMMENT THE ABOVE LINE TO EXECUTE THE CELL ###

### ------------------------------------------------------------------------------- ###
### TRY YOURSELF! THIS IS WHAT WILL HAPPEN WHEN YOU CLICK 'Get Editors Info' BUTTON ###
### ------------------------------------------------------------------------------- ###

## Create a TokensOwnedListener instance, more details can be found in             ##
## https://github.com/gesiscss/wikiwho_demo_new/blob/master/visualization/tokens_listener.py ##
owned_listener = TokensOwnedListener(total_actions, token_sources, lng=abbreviation(languageSelection.value))

## Visualization, core visual code lies in TokensOwnedListener, then the interact function     ##
## makes it interactive, mode details see:                                                 ##
## https://github.com/gesiscss/wikiwho_demo/blob/master/visualization/tokens_listener.py  ##

# This is the page you used
print('The page that is being used:', the_page['title'], f'({abbreviation(languageSelection.value).upper()})')

# Visualization: parameters
begin_date = date(2005, 3, 22)
end_date = date(2021, 2, 26)
frequency = 'Monthly' # "Monthly", "Weekly", "Daily"
stopwords = 'Not included'

# Visualization: the stacked area plot
owned_listener.listen(
    _range1 = begin_date,
    _range2 = end_date,
    granularity = frequency,
    stopwords = stopwords
)
%%script false --no-raise-error

### IMPORTANT NOTE: COMMENT THE ABOVE LINE TO EXECUTE THE CELL ###

### -------------------------------------------------------------------------- ###
### TRY YOURSELF! THIS IS WHAT WILL HAPPEN WHEN YOU CLICK 'Show Tokens' BUTTON ###
### -------------------------------------------------------------------------- ###

## Create a TokensListener instance, more details see:                                       ##
## https://github.com/gesiscss/wikiwho_demo_new/blob/master/visualization/tokens_listener.py ##
tokens_listener = TokensListener(total_actions, token_sources, lng=abbreviation(languageSelection.value))

## Visualization ##
selected_rev = "954896444"
include_stopwords = "Not included"  # included, Not included

tokens_listener.listen(int(selected_rev), include_stopwords)<jupyter_output><empty_output><jupyter_text>A.4 Editor activity drilldown -  Detailed information for each editor in a time frameWe can explore individual editors in detail. The table below presents several metrics for each editor in Monthly/Weekly/Daily time frames.    Columns description of the editors table**General information:**    - **rev_time**: The time frame within which some revisions created by an editor fall. For example, when choosing "Monthly" time frame, "2020-05-31" denotes the whole month of May, 2020; when choosing "Weekly" time frame, "2020-05-10" denotes the week from 2020-05-04 to 2020-05-10 and it is always Sunday; when choosing "Daily" time frame, the "rev_time" simply indicates "on that day".- **editor**: the name of an editor that performed an action.- **editor_id**: the id of the editor that inserted the token (if starts with 0|, it means that the editor is not registered, and the ip is displayed instead)    **Metrics taking stopwords into consideration:**- **adds**: Number of first-time insertions (additions).- **dels**: Neditorsumber of deletions.- **reins**: Number of reinsertions.- **prod**: Productivity, ratio of all actions that survived at least 48h.- **SW_ratio**: Stopwords ratio, ratio of stopwords.- **revs**: Total number of the revisions created by this editor within some time frame.    **Metrics excluding stopwords:**- **conflict**: The sum of conflict scores of all actions divided by the number of elegible actions.- **main_op**: Main opponent, the previous editor who has the most conflict with the editor in this row, within some time frame.- **avg_reac_time**: Average reaction time of all elegible actions (dels + reins) of an editor within some time frame.   You can click each row to to know which revisions a particular editor has created in that month/week/day. Then another table containing all revisions in that time frame with other relevant metrics will be provided underneath.   Columns description of the revisions table   In the second table, the definition of adds/dels/reins/prod/SW_ratio remains the same as the one for the editors table. Here are other metrics included:    - **main_op**: (Excluding stopwords) The previous editor who has the most conflict calculated using actions in this revision.- **min_reac**: (Excluding stopwords) The minimum length of the reaction time in this revision.- **Damaging**: The probability of being damaging edits provided by [ORES](https://www.mediawiki.org/wiki/ORESEdit_quality)- **Goodfaith**: The probability of being non-damaging edits provided by [ORES](https://www.mediawiki.org/wiki/ORESEdit_quality)    Then, click one revision that interests you and this revision will be automatically filled in the input box in the next section (subsection B.1).<jupyter_code>searchRev = widgets.Text(description='Rev_id:')

# create and display the button
button_a2 = widgets.Button(description="Get Editors Info", layout=Layout(width='160px'))

display(button_a2)

def on_click_editors(b):
    global editor_listener
    with out_a2:
        clear_output()
        
        editor_listener = EditorsListener(agg_actions, sources, lng=abbreviation(languageSelection.value), search_widget=searchRev)
        editor_listener.get_infos()
        
        display(md(f"***Page: {the_page['title']} ({abbreviation(languageSelection.value).upper()})***"))

        datepicker_from = widgets.DatePicker(description='Date starts', value=agg_actions['rev_time'].iloc[0], layout=Layout(width="25%"))
        datepicker_to = widgets.DatePicker(description='Date ends', value=agg_actions['rev_time'].iloc[-1], layout=Layout(width="25%"))
        timeframe = widgets.RadioButtons(options=['Monthly', 'Weekly', "Daily"], value='Monthly', description='Granularity:', layout={'width': '50%'})
        interact(editor_listener.listen,
                 _range1=datepicker_from,
                 _range2=datepicker_to,
                 granularity=timeframe)
    
out_a2 = Output()
display(out_a2)

# set the event
button_a2.on_click(on_click_editors)

# trigger the event with the default value
on_click_editors(button_a2)<jupyter_output><empty_output><jupyter_text>Try yourself! This is what will happen when you click 'Load Actions' button:<jupyter_code>%%script false --no-raise-error

### IMPORTANT NOTE: COMMENT THE ABOVE LINE TO EXECUTE THE CELL ###

### ------------------------------------------------------------------------------- ###
### TRY YOURSELF! THIS IS WHAT WILL HAPPEN WHEN YOU CLICK 'Get Editors Info' BUTTON ###
### ------------------------------------------------------------------------------- ###

## Create an EditorsListener instance and calculate, more details can be found in             ##
## https://github.com/gesiscss/wikiwho_demo_new/blob/master/visualization/editors_listener.py ##
editors_listener = EditorsListener(total_actions, token_sources, lng=abbreviation(languageSelection.value))
editors_listener.get_infos()


## Visualization, core visual code lies in EditorsListener, then the interact function     ##
## makes it interactive, mode details see:                                                 ##
## https://github.com/gesiscss/wikiwho_demo/blob/master/visualization/editors_listener.py  ##

# This is the page you used
print('The page that is being used:', the_page['title'], f'({abbreviation(languageSelection.value).upper()})')

# Visualization: parameters
begin_date = date(2007, 3, 1)
end_date = date(2020, 5, 1)
frequency = 'Monthly' # "Monthly", "Weekly", "Daily"

the_latest_date = editors_listener.df.iloc[-1]["rev_time"].date()
the_latest_editor = str(editors_listener.df.iloc[-1]["editor_id"])
selected_date = the_latest_date  # Pick any datetime value from "rev_time" column, example: date(2020, 4, 30)
selected_editor = the_latest_editor  # Pick any string value from "editor_id" column, example: "17723088"

# Visualization: the first table with general info
editors_listener.listen(
    _range1 = begin_date,
    _range2 = end_date,
    granularity = frequency,
)

# Visualization: the second table with revision info
print(f"Within {frequency} timeframe, you have selected {editors_listener.names_id[selected_editor]}(id: {selected_editor})")
print(f"The revisions fall in {str(selected_date)}")
print(f"Link to revision history: https://{editors_listener.lng}.wikipedia.org/w/index.php?date-range-to=\
{str(selected_date)}&tagfilter=&title={the_page['title'].replace(' ', '_')}&action=history")

# Use RevisionsManager, more details see:
# https://github.com/gesiscss/wikiwho_demo_new/blob/master/visualization/editors_listener.py
revisions_manager = editors_listener.revision_manager
rev_table = revisions_manager.get_main(selected_date, selected_editor, frequency)
display(rev_table)

# Visualization: get the comment for this revision
editors_listener.get_comments()  
selected_rev = rev_table["rev_id"].iloc[0]  # The first revision in the second table. You can also input another revision id if more are available.
print(f"Comment for the revision {selected_rev}: {editors_listener.rev_comments[editors_listener.selected_rev]}")
print(f"Link to revisions diff: https://{editors_listener.lng}.wikipedia.org/w/index.php?diff={selected_rev}\
&title=TITLEDOESNTMATTER&diffmode=source")<jupyter_output><empty_output><jupyter_text>B Text change inspection - Which text was changed?  B.1 Text change per revision - Which tokens were changed in a revision?The pre-selected revision corresponds to the latest revision of this article. You can enter a revision id manually, but we recommend to select a revision through the controls in the previous section (subsection A.4). Once you have selected the revisions, you can click "Show tokens" button.<jupyter_code># create and display the button
button21 = widgets.Button(description="Show tokens")

def on_click_rev_tokens(b):
    global listener
    with out212:
        clear_output()    
        listener = TokensListener(agg_actions, sources, lng=abbreviation(languageSelection.value))
        

        # visualization
        search_value = editor_listener.search_widget.value
        #listener.rev_id = int(search_value)
        listener.listen(int(search_value), stopwords_a22.value)
        #out = interactive_output(listener.listen, {'stopwords':stopwords_a22})
        #display(out)
        
        # preset filter to display only the selected revision
        if len(listener.filtered_df) != 0:
            listener.qgrid_selected_revision._handle_qgrid_msg_helper({
                'type': 'show_filter_dropdown',
                'field': 'rev_id',
                'search_val': search_value})

            listener.qgrid_selected_revision._handle_qgrid_msg_helper({
                    'field': "rev_id",
                    'filter_info': {
                        'field': "rev_id",
                        'selected': [0],
                        'type': "text",
                        'excluded': []
                    },
                    'type': "change_filter"
                })
        
selected_rev = copy.copy(editor_listener.selected_rev)
stopwords_a22 = widgets.RadioButtons(options=['Not included', 'Included'], value='Not included', description='Stopwords', layout={'width': '50%'})
display(editor_listener.search_widget)
display(stopwords_a22)
display(button21)

out212 = Output()
display(out212)

# set the event
button21.on_click(on_click_rev_tokens)

# trigger the event with the default value
on_click_rev_tokens(button21)<jupyter_output><empty_output><jupyter_text>Try yourself! This is what will happen when you click 'Show Tokens' button:<jupyter_code>%%script false --no-raise-error

### IMPORTANT NOTE: COMMENT THE ABOVE LINE TO EXECUTE THE CELL ###

### -------------------------------------------------------------------------- ###
### TRY YOURSELF! THIS IS WHAT WILL HAPPEN WHEN YOU CLICK 'Show Tokens' BUTTON ###
### -------------------------------------------------------------------------- ###

## Create a TokensListener instance, more details see:                                       ##
## https://github.com/gesiscss/wikiwho_demo_new/blob/master/visualization/tokens_listener.py ##
tokens_listener = TokensListener(total_actions, token_sources, lng=abbreviation(languageSelection.value))

## Visualization ##
selected_rev = "1005338891"
include_stopwords = "Not included"  # included, Not included

tokens_listener.listen(int(selected_rev), include_stopwords)<jupyter_output><empty_output><jupyter_text>B.2 Text change overall - Which tokens were changed overall?The following WordCloud and table present the 100 most popular token strings based on the amount of times that they are **added** or **undone (deleted/reinserted)**. The table on the bottom presents the detailed actions of each token.        Explanation of the change action types   - **adds**: number of first-time insertions (additions)- **adds_48h**: number of adds that survived at least 48 hours- **adds_48h_ratio**: adds_48h / adds- **dels**: number of deletions- **dels_48h**: number of deletions that were not resinserted in the next 48 hours- **dels_48h_ratio**: dels_48h / dels- **reins**: number of reinsertions- **reins_48h**: number of reinsertions that survived at least 48 hours- **reins_48h_ratio**: reins_48h / reinsIf you are insterested in a particular token, lick the token (row) in the table and another table containing all revision history of this token within the selected date range will appear beneath the first table. After that, you can investigate one particular action implemented on that token, then click that action (revision row) and a link to Wikipedia will be provided below for comparing different revisions. "Conflicts" tabIn the "Conflicts" tab, the WordCloud displays the top-100 conflicting token strings  i.e. the words with the most actions that are related to conflicts. The size of the token string in the WordCloud indicates frequency of actions (by default, only actions with conflict are used as *source* for the wordcloud).        Source description    -   **Only Conflicts**: use only the actions that are in conflict.-   **Elegible Actions**: use only the actions that can potentially enter into conflict, i.e. actions that have occurred at least twice, e.g. the token x has been inserted twice (which necessarily implies it was remove once), the token x has been deleted twice (which necessarily implies it was inserted twice) -   **All Actions**: use all tokens regardles conflict "Both" tabIn the "Both" tab, all the information is displayed simultaneusly in the follwing order: 1. the top-100 conflicting token strings 2. the 100 most popular token strings  3. the interactive table of actions of each token<jupyter_code># create and display the button
button3 = widgets.Button(description="Show WordCloud")
wc_select = widgets.ToggleButtons(options=['All actions', 'Conflicts', 'Both'], description='Based on:',
    tooltips=['Most frequent words changed', 'Most frequent conflicting words', 'Both']
)
display(wc_select, button3)

def on_click_wc_action(b):
    with out12:
        clear_output()
    
        listener = WCActionsListener(sources={
            "tokens_source": sources,
            "tokens_inc_stop": actions_inc_stop
        }, lng=abbreviation(languageSelection.value))
        
        con_listener = WCListener(sources={"tokens_source": sources}, lng=abbreviation(languageSelection.value))
 

        # visualization
        date_from = sources["tokens_all"]["rev_time"].iloc[0].date()
        date_to = sources["tokens_all"]["rev_time"].iloc[-1].date()
        datepicker_from = widgets.DatePicker(description='Date starts', value=date_from, layout=Layout(width='25%'))
        datepicker_to = widgets.DatePicker(description='Date ends', value=date_to, layout=Layout(width='25%'))
        
        _range1=datepicker_from
        _range2=datepicker_to
        action=Dropdown(options=['All', 'adds', 'adds_48h', 'dels', 'dels_48h', 'reins', 'reins_48h'], value='All', description='Action')
        stopwords = widgets.RadioButtons(options=['Not included', 'Included'], value='Not included', description='Stopwords', layout={'width': '50%'})
        
        # conflict tokens visualization
        source=Dropdown(options=['All Actions', 'Elegible Actions', 'Only Conflicts'], value='Only Conflicts', description='Source (*)')
        con_action=Dropdown(options=['Both', 'Just Insertions', 'Just Deletions'], value='Both', description='Action')
        editor=fixed('All')
        
        out = interactive_output(listener.listen, {'_range1': _range1, '_range2':_range2, 'action':action, 'stopwords':stopwords})
        
        con_out = interactive_output(con_listener.listen, {
                 '_range1': datepicker_from,
                 '_range2': datepicker_to,
                 'source': source,
                 'action': con_action,
                 'editor': editor,
                 'stopwords': stopwords
        })

        display(md(f"***Page: {the_page['title']} ({abbreviation(languageSelection.value).upper()})***"))
                   
        if wc_select.value == 'All actions':
            display(VBox([datepicker_from, datepicker_to, action, stopwords, out]))
        elif wc_select.value == 'Conflicts':
            display(VBox([datepicker_from, datepicker_to, con_action, source, stopwords, con_out]))
        else:
            display(VBox([con_action, source, con_out]))
            display(VBox([datepicker_from, datepicker_to, action, stopwords, out]))
            


        

out12 = Output()
display(out12)

# set the event
button3.on_click(on_click_wc_action)
wc_select.observe(on_click_wc_action, 'value')

# trigger the event with the default value
on_click_wc_action(button3)<jupyter_output><empty_output><jupyter_text>Try yourself! This is what will happen when you click 'Show WordCloud' button:<jupyter_code>%%script false --no-raise-error

### IMPORTANT NOTE: COMMENT THE ABOVE LINE TO EXECUTE THE CELL ###

### -------------------------------------------------------------------------- ###
### TRY YOURSELF! THIS IS WHAT WILL HAPPEN WHEN YOU CLICK 'Show Word Cloud' BUTTON ###
### -------------------------------------------------------------------------- ###

wc_action_listener = WCActionsListener(sources={
            "tokens_source": sources,
            "tokens_inc_stop": actions_inc_stop
        }, lng=abbreviation(languageSelection.value))

wc_con_listener = WCListener(sources={"tokens_source": sources}, lng=abbreviation(languageSelection.value))

# Parameters
date_start = date(2005, 3, 22)
date_end = date(2021, 2, 7)
action_type = 'All'  #'All', 'adds', 'adds_48h', 'dels', 'dels_48h', 'reins', 'reins_48h'
stopwords = "Not included" # 'included', 'Not included'

# conflict tokens visualization
source = 'All Actions' #'All Actions', 'Elegible Actions', 'Only Conflicts'
conflict_action = 'Both' #'Both', 'Just Insertions', 'Just Deletions'
editor='All'

wc_action_listener.listen(_range1 = date_start, _range2 = date_end, action = action_type, stopwords = stopwords)

# Uncomment this to plot the word cloud for conflicts        
# wc_con_listener.listen(
#                  _range1 = date_start,
#                  _range2 = date_end,
#                  source = source,
#                  action = conflict_action,
#                  editor = editor,
#                  stopwords = stopwords)<jupyter_output><empty_output><jupyter_text>--- B.3 Token-level conflicts - Conflict scores of individual change actions In the following table, all the actions that are in conflict are presented, and a conflict scoreis presented per action. The importance of the conflict can be meassure by considering the seconds t that have passed since the last action on the same token (string) has occured (`time_diff_secs` column). A score to meassure conflict is calculated based on t with the following formula: 1 / log3600(t+2). Thus, *undo* actions are weighted higher than the original time in seconds when the *t* is less than an hour.For details, please refer to [Flöck et al, 2017](https://arxiv.org/abs/1703.08244).        Columns description- **string**: the string of the token that is being tracked- **order**: numerical variable, shows the order of the conflicting actions in the revision history- **count**: numerical indicator of how often a given token is in the revision history- **token_id**: the id of the token that is being tracked- **rev_id**: the revision id in which the action (insertion or deletion) happen- **editor_name**: the name of an editor that performed an action- **editor_id**: the id of the editor that inserted the token (if starts with **0|**, it means thatthe editor is not registered, and the ip is displayed instead- **time_diff_secs**: seconds that have passed since the last action on the same token has occured- **conflict**: a score to meassure conflict that is calculated based on the `time_diff_secs` with the following formula: *1 / log3600(time_diff_secs + 2)*. For details, please refer to [Flöck et al, 2017](https://arxiv.org/abs/1703.08244)<jupyter_code># create and display the button
button2 = widgets.Button(description="Show Conflict Score of Each Action", layout=Layout(width='250px'))
display(button2)

def on_click_actions(b):
    global conflictaction_listener
    with out9:
        clear_output()
        
        display(md(f"***Page: {the_page['title']} ({abbreviation(languageSelection.value).upper()})***"))
        datepicker_from = widgets.DatePicker(description='Date starts', value=pd.to_datetime(revisions['rev_time'].iloc[0]), layout=Layout(width='25%'))
        datepicker_to = widgets.DatePicker(description='Date ends', value=pd.to_datetime(revisions['rev_time'].iloc[-1]), layout=Layout(width='25%'))
        
        conflictaction_listener = ConflictsActionListener(sources={
            "tokens_source": sources,
            "Editors": editors,
            "Revisions": revisions
        },lng=abbreviation(languageSelection.value))
               
        stopwords = widgets.RadioButtons(options=['Not included', 'Included'], value='Not included', description='Stopwords', layout={'width': '50%'})
        interact(conflictaction_listener.listen_to_interact, _range1 = datepicker_from, _range2 = datepicker_to, stopwords = stopwords)

                      
out9 = Output()
display(out9)

# set the event
button2.on_click(on_click_actions)

# trigger the event with the default value
on_click_actions(button2)<jupyter_output><empty_output><jupyter_text>Try yourself! This is what will happen when you click 'Show Conflict Score of Each Action' button:<jupyter_code>%%script false --no-raise-error

### IMPORTANT NOTE: COMMENT THE ABOVE LINE TO EXECUTE THE CELL ###

### --------------------------------------------------------------------------------------------------- ###
### TRY YOURSELF! THIS IS WHAT WILL HAPPEN WHEN YOU CLICK 'Show Conflict Score of Each Action' BUTTON   ###
### --------------------------------------------------------------------------------------------------- ###

## This is the page you used ##
print('The page that is being used:', the_page['title'], f'({abbreviation(languageSelection.value).upper()})')

## Create a ConflictsActionListener instance, more details see: ##
## https://github.com/gesiscss/wikiwho_demo_new/blob/master/visualization/conflicts_listener.py#L178 ##
# conflict_actions = ConflictsActionListener(sources={
#             "tokens_source": token_sources,
#             "Editors": grabbed_names,
#             "Revisions": all_revisions
#         },lng=abbreviation(languageSelection.value))

## Visualization ##
begin_date = date(2007, 3, 1)
end_date = date(2020, 5, 1)
include_stopwords = "Not included"  # "Not inclueded", "Included"
revisionID = "706448515"  # Any revision id from rev_id column

conflictaction_listener.listen(begin_date, end_date, include_stopwords)
print(f"Link to the wikipedia diff: https://{conflictaction_listener.lng}.wikipedia.org/w/index.php?&title={the_page['title'].replace(' ', '_')}&diff={revisionID}")<jupyter_output><empty_output><jupyter_text>--- C. Wikipedia Talk pageIn this section, information about the associated Talk Page is presented. [Talk pages](https://en.wikipedia.org/wiki/Help:Talk_pages) are "administration pages where editors can discuss improvements to articles". In practice, Talk pages are also Wikipedia articles that can be accessed by adding `Talk:` before the name label in the url; for example, the talk page for https://en.wikipedia.org/wiki/The_Camp_of_the_Saints is called https://en.wikipedia.org/wiki/Talk:The_Camp_of_the_Saints.  C.1 Talk frequency - How are revisions in the talk section distributed over time?The following graph displays the number of revisions on the Talk page that is associated to the article that is being analyzed. This information is extracted directly from the [Wikipedia API](https://en.wikipedia.org/w/api.php). This information is also relevant to understand the conversations around conflictive periods that are observable in the time seried displayed in this notebook.<jupyter_code>button_t = widgets.Button(description="Load Talks", layout=Layout(width='160px'))
display(button_t)

def on_click_details(b):
    global talk_content 
    with out_t:
        clear_output()
        display(md(f"***Page: {the_page['title']} ({abbreviation(languageSelection.value).upper()})***"))

        #Initiate a Wikipedia instance
        wikipediadv_instance = WikipediaDV(WikipediaAPI(lng=abbreviation(languageSelection.value)))
        
        #Get content of revisions on talk page
        talk_content = wikipediadv_instance.get_talk_content(the_talk_page["page_id"])
        
        #convert to datetime format
        talk_content['year_month'] = pd.to_datetime(talk_content['timestamp'])

        
        # Define a TalksListener instance
        talks_listener = TalksListener(talk_content)

        # Visualization
        datepicker_from = widgets.DatePicker(description='Date starts', value=talk_content['year_month'].iloc[-1], layout=Layout(width='25%'))
        datepicker_to = widgets.DatePicker(description='Date ends', value=talk_content['year_month'].iloc[0], layout=Layout(width='25%'))

        interact(talks_listener.listen,
            begin = datepicker_from,
            end = datepicker_to,
            granularity = Dropdown(options=['Monthly', 'Yearly', 'Daily'], value='Monthly')
        )

out_t = Output()
display(out_t)

# set the event
button_t.on_click(on_click_details)

# trigger the event with the default value
on_click_details(button_t)<jupyter_output><empty_output><jupyter_text>Try yourself! This is what will happen when you click 'Load Talks' button:<jupyter_code>%%script false --no-raise-error

### IMPORTANT NOTE: COMMENT THE ABOVE LINE TO EXECUTE THE CELL ###

### -------------------------------------------------------------------------- ###
### TRY YOURSELF! THIS IS WHAT WILL HAPPEN WHEN YOU CLICK 'Load Talks' BUTTON  ###
### -------------------------------------------------------------------------- ###

## This is the page you used ##
print('The page that is being used:', the_talk_page['title'])

# Initiate a Wikipedia instance
wikipediadv_instance = WikipediaDV(WikipediaAPI(lng=abbreviation(languageSelection.value)))

# Get content of revisions on talk page
# more details see: https://github.com/gesiscss/wikiwho_demo_new/blob/master/external/wikipedia.py#L109
talk_content = wikipediadv_instance.get_talk_content(the_talk_page["page_id"])

# convert to datetime format
talk_content['year_month'] = pd.to_datetime(talk_content['timestamp'])

## Define a TalksListener instance ##
# more details see: https://github.com/gesiscss/wikiwho_demo_new/blob/master/visualization/talks_listener.py
talks_listener = TalksListener(talk_content)

## Visualization ##
begin_date = date(2005, 3, 1)
end_date = date(2015, 7, 4)
granularity = 'Monthly'

print('Time range from', begin_date.strftime("%Y-%m-%d"), 'to', end_date.strftime("%Y-%m-%d"))
talks_listener.listen(
    begin = begin_date,
    end = end_date,
    granularity = granularity)<jupyter_output><empty_output><jupyter_text>C.2 Talk topics - What are talk entries about?The topic of the talks under which the revisions happened are also available. The first table below shows the topic of each revision, sorted by date.  Although most of the times users change the content of the talk page, sometimes revisions include actions like signing a comment, deleting or reversing revisions. These actions are not presented in the table.        Columns description- **topic**: the string topic extracted from revision content- **revid**: the revision id in which the action happened- **user**: the id or ip address of the editor that inserted the token- **year_month**: timestamp when the revision took place- **action_type**: type of the action made in a revision (new topic added or topic edited)<jupyter_code>button_to = widgets.Button(description="Load Topics", layout=Layout(width='160px'))
display(button_to)

def on_click_topics(b):
    global topics_listener, topic_df
    with out_to:
        clear_output()
        display(md(f"***Page: {the_talk_page['title']} ({abbreviation(languageSelection.value).upper()})***"))

        topics_listener = TopicsListener(talk_content)

        #get a dataframe with topics and action_types
        wikipediadv_ins = WikipediaDV(WikipediaAPI(lng=abbreviation(languageSelection.value)))
        topic_df = topics_listener.extract_topics(wikipediadv_ins)
        
                   
        #adding transcluded content
        tr_row = talk_content[talk_content['comment'].str.match(r'Transcluding GA review')==True]
        if len(tr_row)>0:
            wikipedia_dv = WikipediaDV(WikipediaAPI(lng=abbreviation(languageSelection.value)))
            transcluded_page = wikipedia_dv.get_page(the_talk_page['title'] +'/GA1')
            transcluded_talk_content = wikipedia_dv.get_talk_content(transcluded_page["page_id"])
            transcluded_talk_content['year_month'] = pd.to_datetime(transcluded_talk_content['timestamp'])
            tr_topics_listener = TopicsListener(transcluded_talk_content)
            transcluded_topic_df = tr_topics_listener.translusion(wikipedia_dv).set_index('topics')
            topics_listener.topic_df = topic_df.append(transcluded_topic_df).sort_values(by='year_month', ascending = False)
        if len(topic_df) == 0:
            display(md("There is no topics here!"))
        else:        
            display(qgrid.show_grid(topics_listener.topic_df))

            # Visualization
            datepicker_from = widgets.DatePicker(description='Date starts', value=talk_content['year_month'].iloc[-1], layout=Layout(width='25%'))
            datepicker_to = widgets.DatePicker(description='Date ends', value=talk_content['year_month'].iloc[0], layout=Layout(width='25%'))

            interact(topics_listener.listen,
                begin = datepicker_from,
                end = datepicker_to,
                granularity = Dropdown(options=['Monthly', 'Yearly', 'Daily'], value='Monthly')
            )

out_to = Output()
display(out_to)

# set the event
button_to.on_click(on_click_topics)

# trigger the event with the default value
on_click_topics(button_to)<jupyter_output><empty_output><jupyter_text>Try yourself! This is the code executed when you click the 'Load Topics' button:<jupyter_code>%%script false --no-raise-error

### IMPORTANT NOTE: COMMENT THE ABOVE LINE TO EXECUTE THE CELL ###

### --------------------------------------------------------------------------- ###
### TRY YOURSELF! THIS IS WHAT WILL HAPPEN WHEN YOU CLICK 'Show Topics' BUTTON  ###
### --------------------------------------------------------------------------- ###

## This is the page you used ##
print('The page that is being used:', the_talk_page['title'])
print(f"Time range from {talk_content['year_month'].iloc[-1].date()} to {talk_content['year_month'].iloc[0].date()}")

## Initiate a topics listener, more details see:                                             ##
## https://github.com/gesiscss/wikiwho_demo_new/blob/master/visualization/topics_listener.py ##
topics_listener = TopicsListener(talk_content)

# Get a dataframe with topics and action_types
wikipediadv_ins = WikipediaDV(WikipediaAPI(lng=abbreviation(languageSelection.value)))
topic_df = topics_listener.extract_topics(wikipediadv_ins)
if len(topic_df) == 0:
    print("There is no topics here!")
else:
    display(qgrid.show_grid(topic_df))

    # Visualization
    begin_date = talk_content['year_month'].iloc[-1]  # date(xxxx.xx.xx)
    end_date = talk_content['year_month'].iloc[0]  # date(xxxx.xx.xx)
    granularity = 'Monthly'

    topics_listener.listen(
        begin = begin_date,
        end = end_date,
        granularity = granularity)<jupyter_output><empty_output><jupyter_text>[Go to next workbook](./3_Focus_on_editor.ipynb)<jupyter_code>lng_selected = abbreviation(languageSelection.value)
#calculator = con_manager
%store agg_actions
#%store calculator
#%store editors_conflicts
%store lng_selected
%store sources
%store con_manager_all

clear_output()
# display(HTML(f'<a href="{get_next_notebook()}" target="_blank">Go to next workbook</a>'))
# if len(editors_conflicts) > 0:
#     display(HTML(f'<a href="{get_next_notebook()}" target="_blank">Go to next workbook</a>'))
# else:
#     display(HTML(f'<a href="{get_previous_notebook()}" target="_blank">Go back to the previous workbook</a>'))



scroll_to_top = """
<script>
document.getElementById('notebook').scrollIntoView();
</script>
"""

display(HTML(scroll_to_top))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12731625568589897, 'fraction_numerical': 0.010294469715106535, 'mean_word_length': 3.65359477124183, 'pattern_counts': {'":': 26, '<': 108, '<?xml version=': 0, '>': 110, 'https://': 38, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 31, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '203854', 'license': 'MIT', 'n_tokens_mistral': 18060, 'n_tokens_neox': 17139, 'n_words': 5357, 'path': '2_Page_analysis.ipynb', 'repo_name': 'gesiscss/IWAAN', 'size': '62661'}