<commit_before>#!/usr/bin/env python
"""
Get plain text of article's lead section from titles or file

INPUT
    Wikipedia article titles or filename
    input file expected to be JSON from Mediawiki API

OUTPUT
    Summary as wikitext, dict, or json

References
    https://en.wikipedia.org/wiki/Wikipedia:Manual_of_Style/Lead_section
    https://www.mediawiki.org/wiki/API:Main_page
    https://meta.wikimedia.org/wiki/Wiki_syntax
"""

from __future__ import print_function

__author__ = "@siznax"
__version__ = "15 Sep 2015"<commit_msg>added more details to wp_summary.py docstring.<commit_after>#!/usr/bin/env python
"""
Get plain text of article's lead section from titles or file

This script has two major features:

    1) Get the article wikitext and extract just the paragraphs in the
       so-called "lead section", and
    2) Clean (as well as possible) the wiki syntax and raw HTML from
       the lead section paragraphs

Currently, an attempt is made to "clean" wikitext (see _pidgin methods
below), but this doesn't give us the best result. We'll need to run
the wikitext through a MediaWiki instance (or script) and then use
lxml to strip tags. We'll probably still need to do some cleanup after
that for linked phrases that are no longer relevent, e.g. "in other
languages" in the IPA template.

    lxml strip_tags() or
        ``lxml.etree.tostring(elem, method="text", encoding="utf-8")``

INPUT
    Wikipedia article titles or filename
    input file expected to be JSON from Mediawiki API

OUTPUT
    Summary as wikitext, dict, or json

References
    https://en.wikipedia.org/wiki/Wikipedia:Manual_of_Style/Lead_section
    https://www.mediawiki.org/wiki/API:Main_page
    https://meta.wikimedia.org/wiki/Wiki_syntax
"""

from __future__ import print_function

__author__ = "@siznax"
__version__ = "15 Sep 2015"
{'fraction_non_alphanumeric': 0.0789760348583878, 'fraction_numerical': 0.008169934640522876, 'mean_word_length': 3.9514824797843664, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3570579', 'n_tokens_mistral': 575, 'n_tokens_neox': 544, 'n_words': 212}