<jupyter_start><jupyter_text>https://stackoverflow.com/questions/34411896/how-to-get-full-wikipedia-revision-history-list-from-some-articlehttp://en.wikipedia.org/w/api.php?action=query&prop=revisions&rvprop=content&format=json&&titles=Akiko_Iwasaki        https://en.wikipedia.org/w/api.php?action=query&format=json&titles=Bla_Bla_Bla&prop=extracts&exintro&explaintext        exintro - parse first paragraph  Get all revisions of Wikipedia articles  Non awarded scientists<jupyter_code>#pip install pywikibot
#create file user-config.py
#
## -*- coding: utf-8 -*-
#family = 'wikipedia'
#mylang = 'en'
#usernames['wikipedia']['en'] = u'RevisionsBot'
#
#
#
#https://doc.wikimedia.org/pywikibot/api_ref/pywikibot.html#module-pywikibot.page

import pywikibot
site = pywikibot.Site("en", "wikipedia")
list_rev=[]
erorr_list=[]
j=1
for sci_name in sci_nonaw_dict.keys():
    name=sci_name.split("/")[-1]
    print j,name
    try:
        page = pywikibot.Page(site, name)
        revs = page.revisions(content=True)
        for i in revs:
            temp_dict=i.__dict__
            temp_dict["article"]=name
            list_rev.append(temp_dict)
    except:
        print "Error when access:", name
        erorr_list.append(name)
        
    j+=1
print erorr_list
df=pd.DataFrame(list_rev)
df.info()
df.text=df.text.apply(lambda x: " ".join(x.split()) if (x==x)&(type(x)==unicode) else None)
df.to_csv("baseline_revisions.csv",sep="\t")<jupyter_output><empty_output><jupyter_text>Count words in revision text<jupyter_code>df["text_len"]=df.text.apply(lambda x: len(TextBlob(x).words)  if (x==x)&(type(x)==unicode) else None)
df.head(2)
df.to_csv("baseline_revisions.csv",sep="\t")<jupyter_output><empty_output><jupyter_text>Awarded scientists<jupyter_code>list_rev2=[]
erorr_list2=[]

j=1
for sci_name in sci_aw_dict.keys():
    name=sci_name.split("/")[-1]
    print j,name
    try:
        page = pywikibot.Page(site, name)
        revs = page.revisions(content=True)
        for i in revs:
            temp_dict=i.__dict__
            temp_dict["article"]=name
            list_rev2.append(temp_dict)
    except:
        print "Error when access:", name
        erorr_list2.append(name)
        
    j+=1
    
df2=pd.DataFrame(list_rev2)
df2.info()

df2.text=df2.text.apply(lambda x: " ".join(x.split()) if (x==x)&(type(x)==unicode) else None)
df2.to_csv("seed_revisions.csv",sep="\t")
print erorr_list2
df2["text_len"]=df2.text.apply(lambda x: len(TextBlob(x).words)  if (x==x)&(type(x)==unicode) else None)
df2.to_csv("seed_revisions.csv",sep="\t")<jupyter_output><empty_output><jupyter_text>convert wiki to plain text<jupyter_code>#pip install mediawiki-parser
#https://github.com/peter17/mediawiki-parser 



from mediawiki_parser.preprocessor import make_parser
from mediawiki_parser.text import make_parser

templates = {}
preprocessor = make_parser(templates)
parser = make_parser()



df["text_clean"]=df.text.apply(lambda x: parser.parse(preprocessor.parse(x).leaves()).value.strip())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15399867812293458, 'fraction_numerical': 0.010575016523463317, 'mean_word_length': 4.554128440366973, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '259610', 'license': 'MIT', 'n_tokens_mistral': 1161, 'n_tokens_neox': 1119, 'n_words': 163, 'path': 'Collect all revisions of Wikipedia articles.ipynb', 'repo_name': 'gesiscss/scientists-analysis-wikipedia', 'size': '3026'}