'''
Created on Mar 7, 2016

@author: Martin Koerner <info@mkoerner.de>
'''

import json

import argparse

from random import shuffle

# generate help text for arguments
parser = argparse.ArgumentParser(description='Extracts domains from a file containing the URLs of Wikipedia articles.')
parser.add_argument('input',
                   help='a file path to the output file generated by this program')
parser.add_argument("--output", dest="output", help='output path', type=str, required=True)
parser.add_argument("--number", dest="number", help='articles to filter out', type=int, required=True)
args = parser.parse_args()

inputfile_path = args.input
outputfile_path = args.output
number_of_articles = args.number

wikipedia_articles = []

# load json input
with open(inputfile_path) as json_input:    
    json_data = json.load(json_input)


# get list of wikipedia article names
for article in json_data:
    wikipedia_articles.append(article)

# shuffle the wikipedia articles
shuffle(wikipedia_articles)

# get first elements specified by number_of_articles
filtered_wikipedia_articles=wikipedia_articles[:number_of_articles]

article_url_dictionary = {}

# iterate over Wikipedia articles and selected the filtered articles
for article in json_data:
    if article in filtered_wikipedia_articles:
        article_url_dictionary[article]=json_data[article] 

# write dictionary with key = Wikipedia article and value = URLs to JSON file
with open(outputfile_path, 'w') as f:
    json.dump(article_url_dictionary, f, indent=4, sort_keys=True)
    print "JSON file was stored successfully"



'''
Created on Mar 7, 2016

@author: Martin Koerner <info@mkoerner.de>
'''

import json

import argparse

from random import shuffle

# generate help text for arguments
parser = argparse.ArgumentParser(description='Extracts domains from a file containing the URLs of Wikipedia articles.')
parser.add_argument('input',
                   help='a file path to the output file generated by this program')
parser.add_argument("--output", dest="output", help='output path', type=str, required=True)
parser.add_argument("--number", dest="number", help='articles to filter out', type=int, required=True)
args = parser.parse_args()

inputfile_path = args.input
outputfile_path = args.output
number_of_articles = args.number

wikipedia_articles = []

# load json input
with open(inputfile_path) as json_input:    
    json_data = json.load(json_input)


# get list of wikipedia article names
for article in json_data:
    if len(json_data[article])>0:
        wikipedia_articles.append(article)

# shuffle the wikipedia articles
shuffle(wikipedia_articles)

# get first elements specified by number_of_articles
filtered_wikipedia_articles=wikipedia_articles[:number_of_articles]

article_url_dictionary = {}

# iterate over Wikipedia articles and selected the filtered articles
for article in json_data:
    if article in filtered_wikipedia_articles:
        article_url_dictionary[article]=json_data[article] 

# write dictionary with key = Wikipedia article and value = URLs to JSON file
with open(outputfile_path, 'w') as f:
    json.dump(article_url_dictionary, f, indent=4, sort_keys=True)
    print "JSON file was stored successfully"



Add check of number of urls in article to wikipedia_article_filtering