<jupyter_start><jupyter_text>English Wikipedia page views, 2008 - 2017  Step 1: Data acquisition Collect dataset of monthly traffic on English Wikipedia from July 1 2008 through September 30 2017.Using Legacy Pagcounts API - From January 2008 through July 2016.     * By Desktop        -- pagecounts_desktop-site_200801-201607.json    * By Mobile         -- pagecounts_mobile-site_200801-201607.json        Variable  :        -- pcStart = 2008010100                -- pcEnd = 2016080100<jupyter_code># Required Library 
import requests
import json
# API  :  https://wikimedia.org/api/rest_v1/#/Legacy_data 
# Understanding of Legacy data API  - Modifying endpoint and Parameter accordingly 
endpoint = 'https://wikimedia.org/api/rest_v1/metrics/legacy/pagecounts/aggregate/{project}/{access-site}/{granularity}/{start}/{end}'
headers={'User-Agent' : 'https://github.com/abhishekanand', 'From' : '<EMAIL>'}

#    -- pcStart = 2008010100     
#  -- pcEnd = 2016080100
       

# Getting PageCount [Legacy API (User+Spider)]  : Desktop-site 
OUTPUT_File = 'pagecounts_desktop-site_200801-201607.json'
params = {'project' : 'en.wikipedia.org',
            'access-site' : 'desktop-site',
            'granularity' : 'monthly',
            'start' : '2008010100',
            'end' : '2016080100'#use the first day of the following month to ensure a full month of data is collected
            }

api_call = requests.get(endpoint.format(**params))
#print(endpoint.format(**params))
pagecount_desktop_views = api_call.json()
#print(pagecount_desktop_views) # Print Resposne from API call 

# https://stackoverflow.com/questions/17043860/python-dump-dict-to-json-file
with open(OUTPUT_File, 'w') as fp:
    json.dump(pagecount_desktop_views, fp)
    
# Getting PageCount [Legacy API (User+Spider)]   : Mobile-site
OUTPUT_File = 'pagecounts_mobile-site_200801-201607.json'
params = {'project' : 'en.wikipedia.org',
            'access-site' : 'mobile-site',
            'granularity' : 'monthly',
            'start' : '2008010100',
            'end' : '2016080100'#use the first day of the following month to ensure a full month of data is collected
            }

api_call = requests.get(endpoint.format(**params))
#print(endpoint.format(**params))
pagecount_mobile_views = api_call.json()
#print(pagecount_mobile_views) # Print Resposne from API call 

# https://stackoverflow.com/questions/17043860/python-dump-dict-to-json-file
with open(OUTPUT_File, 'w') as fp:
    json.dump(pagecount_mobile_views, fp)<jupyter_output><empty_output><jupyter_text>Using Pageviews API - From July 2015 through September 2017.* By Desktop    -- pageviews_desktop-site_201501-201709.json       * By Mobile web    -- pageviews_mobileweb-site_201501-201709.json* By Mobile App    -- pageviews_mobileapp-site_201501-201709.json<jupyter_code># API  : https://wikimedia.org/api/rest_v1/#/Pageviews_data
# Understanding of PageViews API  - Modifying endpoint and Parameter accordingly 
endpoint = 'https://wikimedia.org/api/rest_v1/metrics/pageviews/aggregate/{project}/{access}/{agent}/{granularity}/{start}/{end}'
headers={'User-Agent' : 'https://github.com/abhishekanand', 'From' : '<EMAIL>'}

# Using Pageviews API - From July 2015 through September 2017
#pvStart = 2015070100
#pvEnd = 2017091000 

# Getting PageView [(User)] : Desktop
OUTPUT_File = 'pageviews_desktop-site_201501-201709.json'
params = {'project' : 'en.wikipedia.org',
            'access' : 'desktop',
            'agent' : 'user',
            'granularity' : 'monthly',
            'start' : '2015070100',
            'end' : '2017100100'#use the first day of the following month to ensure a full month of data is collected
            }

api_call = requests.get(endpoint.format(**params))
pageview_desktop_views = api_call.json()
# print(pageview_desktop_views) # Print Resposne from API call 

# https://stackoverflow.com/questions/17043860/python-dump-dict-to-json-file
with open(OUTPUT_File, 'w') as fp:
    json.dump(pageview_desktop_views, fp)
    
# Getting PageView [(User)]  : mobile-app
OUTPUT_File = 'pageviews_mobileapp-site_201501-201709.json'
params = {'project' : 'en.wikipedia.org',
            'access' : 'mobile-app',
            'agent' : 'user',
            'granularity' : 'monthly',
            'start' : '2015070100',
            'end' : '2017100100'#use the first day of the following month to ensure a full month of data is collected
            }

api_call = requests.get(endpoint.format(**params))
pageview_mobile_app_views = api_call.json()
# print(pageview_mobile_app_views) # Print Resposne from API call 

# https://stackoverflow.com/questions/17043860/python-dump-dict-to-json-file
with open(OUTPUT_File, 'w') as fp:
    json.dump(pageview_mobile_app_views, fp)
# Getting PageView [(User)]  : mobile-web
OUTPUT_File = 'pageviews_mobileweb-site_201501-201709.json'
params = {'project' : 'en.wikipedia.org',
            'access' : 'mobile-web',
            'agent' : 'user',
            'granularity' : 'monthly',
            'start' : '2015070100',
            'end' : '2017100100'#use the first day of the following month to ensure a full month of data is collected
            }

api_call = requests.get(endpoint.format(**params))
pageview_mobile_web_views = api_call.json()
# print(pageview_mobile_web_views) # Print Resposne from API call 

# https://stackoverflow.com/questions/17043860/python-dump-dict-to-json-file
with open(OUTPUT_File, 'w') as fp:
    json.dump(pageview_mobile_web_views, fp)<jupyter_output><empty_output><jupyter_text>Step 2: Data processing<jupyter_code>#From Jan 1 2008 to Sep 2017,
# output dictionary will have 12*9+9=117

# creating output dictionary with year, month and all other data set as 0


startYear = 2008
startMonth = 1
endYear = 2017
#endMonth = 9

#totalMonths = (endYear-1-startYear+1)*12 + endMonth
#print(totalMonths)

output = []

for year in range(startYear, endYear+1):
        for month in range(startMonth, 13):
            temp ={}
            temp["year"] = str(year)
            # https://stackoverflow.com/questions/339007/nicest-way-to-pad-zeroes-to-string
            temp["month"] = str(month).rjust(2, '0')
            temp["pagecount_all_views"] = 0
            temp["pagecount_desktop_views"] = 0
            temp["pagecount_mobile_views"] = 0
            temp["pageview_all_views"] = 0
            temp["pageview_desktop_views"] = 0
            temp["pageview_mobile_views"] = 0
            output.append(temp)
#print(output)


# Code 
# pagecount_desktop_views
count_pagecount_desktop_views = 0 
for item in pagecount_desktop_views["items"]:
    timestamp = item["timestamp"]
    
    # parsing timestamp to get year and month
    year = timestamp[0:4] 
    month = timestamp[4:6]
    
    views = item["count"]
    
    
    for element in output:
        if(element["year"]==year and element["month"]==month):
            count_pagecount_desktop_views= count_pagecount_desktop_views+1
            element["pagecount_desktop_views"] = views
            
#print(output)
#print(count_pagecount_desktop_views)
# Code 
# pagecount_mobile_views
count_pagecount_mobile_views = 0 
for item in pagecount_mobile_views["items"]:
    timestamp = item["timestamp"]
    
    # parsing timestamp to get year and month
    year = timestamp[0:4] 
    month = timestamp[4:6]
    
    views = item["count"]
    
    
    for element in output:
        if(element["year"]==year and element["month"]==month):
            count_pagecount_mobile_views= count_pagecount_mobile_views+1
            element["pagecount_mobile_views"] = views
            
#print(output)
#print(count_pagecount_mobile_views)

# Code 
#(pageview_desktop_views


count_pageview_desktop_views = 0 
for item in pageview_desktop_views["items"]:
    timestamp = item["timestamp"]
    
    # parsing timestamp to get year and month
    year = timestamp[0:4] 
    month = timestamp[4:6]
    
    views = item["views"]
    
    
    for element in output:
        if(element["year"]==year and element["month"]==month):
            count_pageview_desktop_views= count_pageview_desktop_views+1
            element["pageview_desktop_views"] = views
            
#print(output)
#print(count_pageview_desktop_views)


#(pageview_mobile_app


count_pageview_mobile_app_views = 0 
for item in pageview_mobile_app_views["items"]:
    timestamp = item["timestamp"]
    
    # parsing timestamp to get year and month
    year = timestamp[0:4] 
    month = timestamp[4:6]
    
    views = item["views"]
    
    
    for element in output:
        if(element["year"]==year and element["month"]==month):
            count_pageview_mobile_app_views= count_pageview_mobile_app_views+1
            element["pageview_mobile_views"] = views
            
#print(output)
#print(count_pageview_mobile_app_views)
#  Code 
#pageview_mobile_web
count_pageview_mobile_web_views = 0 
for item in pageview_mobile_web_views["items"]:
    timestamp = item["timestamp"]
    
    # parsing timestamp to get year and month
    year = timestamp[0:4] 
    month = timestamp[4:6]
    
    views = item["views"]
    
    
    for element in output:
        if(element["year"]==year and element["month"]==month):
            count_pageview_mobile_web_views= count_pageview_mobile_web_views+1
            # For data collected from the Pageviews API, combine the monthly values for mobile-app and mobile-web to create a total mobile traffic count for each month.
            element["pageview_mobile_views"] =  element["pageview_mobile_views"] + views
            
#print(output)
#print(count_pageview_mobile_web_views)

# pagecount_all_views = pagecount_mobile_views + pagecount_desktop_views
# pageview_all_views = pageview_mobile_views + pageview_desktop_views
for item in output:
    item["pagecount_all_views"] = item["pagecount_mobile_views"] + item["pagecount_desktop_views"]
    item["pageview_all_views"] = item["pageview_mobile_views"] + item["pageview_desktop_views"]

#print(output)
# The final data file should be named: en-wikipedia_traffic_200801-201709.csv
import csv
CSVOut = "en-wikipedia_traffic_200801-201709.csv"
with open(CSVOut, "w") as ofile:
    ofile.write("year" + "," + "month" + "," + "pagecount_all_views"+"," + "pagecount_desktop_views"+"," + "pagecount_mobile_views"+"," + "pageview_all_views"+"," + "pageview_desktop_views"+"," + "pageview_mobile_views"+ "\n")
    for item in output:
        ofile.write(str(item["year"]) + "," + str(item["month"]) + "," + str(item["pagecount_all_views"])+"," + str(item["pagecount_desktop_views"])+"," + str(item["pagecount_mobile_views"]) +"," + str(item["pageview_all_views"]) +"," + str(item["pageview_desktop_views"]) +"," + str(item["pageview_mobile_views"]) + "\n")
# print("Done")<jupyter_output><empty_output><jupyter_text>Step 3: Analysis<jupyter_code># Creating Dataframe to Create Vizalization 
#https://chrisalbon.com/python/pandas_dataframe_importing_csv.html
import pandas as pd
import numpy as np

df = pd.DataFrame(output, columns=['year','month','pagecount_all_views','pagecount_desktop_views','pagecount_mobile_views','pageview_all_views','pageview_desktop_views','pageview_mobile_views'])
#print(df)

# Python Pandas replace multiple columns zero to Nan
# https://stackoverflow.com/questions/45416684/python-pandas-replace-multiple-columns-zero-to-nan

df = df.replace({0:np.nan})
#print(df)
# Pandas Documnetation 
# https://pandas.pydata.org/pandas-docs/stable/visualization.html 

# Convert Select Columns in Pandas Dataframe to Numpy Array
# https://stackoverflow.com/questions/31789160/convert-select-columns-in-pandas-dataframe-to-numpy-array
# df.as_matrix(columns=df.columns[2:])

import matplotlib.pyplot as plt
import matplotlib
matplotlib.style.use('ggplot')
%matplotlib inline



dfg = pd.DataFrame(df.as_matrix(columns=df.columns[2:]), index=pd.date_range(start='1/1/2008', end='12/1/2017', freq='MS'), columns=['pagecount_all_views','pagecount_desktop_views','pagecount_mobile_views','pageview_all_views','pageview_desktop_views','pageview_mobile_views'])

#print(dfg)

plt.figure(num=1,figsize=(16,6)) # https://stackoverflow.com/questions/332289/how-do-you-change-the-size-of-figures-drawn-with-matplotlib

plt.plot(dfg )
plt.xlabel('Year')
plt.ylabel('View Count [1e10]')

plt.title('Page Views on English Wikipedia [ January 1 2008 through September 30 2017 ]')
plt.legend(['TotalViews','DesktopViews','MobileViews','TotalViews-UserOnly','DesktopViews -UserOnly','MobileViews-User Only'])
plt.savefig('Mobile and desktop traffic data for English WIkipedia from 2008-2016.png')
plt.savefig('Mobile and desktop traffic data for English WIkipedia from 2008-2016.jpeg')
plt.savefig('Mobile and desktop traffic data for English WIkipedia from 2008-2016.pdf')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1243824013802839, 'fraction_numerical': 0.04329072229629049, 'mean_word_length': 3.9949079514296906, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 17, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '285891', 'license': 'MIT', 'n_tokens_mistral': 4519, 'n_tokens_neox': 4020, 'n_words': 838, 'path': 'hcds-a1-data-curation.ipynb', 'repo_name': 'abhishekanand/data-512-a1', 'size': '12763'}