<commit_before># -*- coding: utf-8 -*-
"""
Loads wikiproject.json, validates it, stores it
Copyright (C) 2015 <NAME>
Licensed under MIT License: http://mitlicense.org
"""

import os
import configparser
import json
import mw
from bs4 import BeautifulSoup
from project_index import WikiProjectTools


def main():

    loginfile = configparser.ConfigParser()
    loginfile.read([os.path.expanduser('~/.wiki.ini')])
    username = loginfile.get('wiki', 'username')
    password = loginfile.get('wiki', 'password')

    enwp = mw.Wiki('https://en.wikipedia.org/w/api.php')
    enwp.login(username, password)

    # Exports the contents of the wikiproject.json page
    params = {'action': 'query', 'format': 'json', 'titles': 'Wikipedia:WikiProject X/wikiproject.json', 'export': ''}
    data = enwp.request(params)

    dump = data['query']['export']['*']

    # We now have a calzone with JSON filling in an XML crust. Boy is this stupid.
    output = BeautifulSoup(dump, 'xml')
    output = output.find('text')  # The contents are stored in an XML field called 'text'
    output = output.get_text()

    # We now have the JSON blob, in string format.
    try:
        output = json.loads(output)
    except ValueError:  # If JSON is invalid
        print('Bad JSON! Bad JSON!')

    print(output)

if __name__ == "__main__":
    main()<commit_msg>Generates an error log if JSON is invalid and kills the script<commit_after># -*- coding: utf-8 -*-
"""
Loads wikiproject.json, validates it, stores it
Copyright (C) 2015 <NAME>
Licensed under MIT License: http://mitlicense.org
"""

import os
import configparser
import json
import mw
from bs4 import BeautifulSoup
from project_index import WikiProjectTools


def main():

    loginfile = configparser.ConfigParser()
    loginfile.read([os.path.expanduser('~/.wiki.ini')])
    username = loginfile.get('wiki', 'username')
    password = loginfile.get('wiki', 'password')

    enwp = mw.Wiki('https://en.wikipedia.org/w/api.php')
    enwp.login(username, password)

    # Exports the contents of the wikiproject.json page
    params = {'action': 'query', 'format': 'json', 'titles': 'Wikipedia:WikiProject X/wikiproject.json', 'export': ''}
    data = enwp.request(params)

    dump = data['query']['export']['*']

    # We now have a calzone with JSON filling in an XML crust. Boy is this stupid.
    output = BeautifulSoup(dump, 'xml')
    output = output.find('text')  # The contents are stored in an XML field called 'text'
    output = output.get_text()

    # We now have the JSON blob, in string format.
    try:
        output = json.loads(output)
    except ValueError as ack:  # If JSON is invalid
        now = datetime.datetime.utcnow()
        wikitime = now.strftime('%Y%m%d%H%M%S')
        report = wikitime + ': ' + ack
        filename = "errors.log"
        save = open(filename, "w")
        save.write(report)
        save.close()
        sys.exit()

    print(output)

if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.13598115112756648, 'fraction_numerical': 0.004039044092898014, 'mean_word_length': 3.36417033773862, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6097392', 'n_tokens_mistral': 956, 'n_tokens_neox': 928, 'n_words': 281}