<commit_before>#!pi_values/usr/bin/python3.5
 
#Author:        <NAME>
#Date:          5/1/17
#Organization:  Commerce Data Service
#Description:   This is a bot script for getting Census data from the Census
#Bureau API's and writing it to Wikipedia.

import pywikibot, json, os, requests, argparse, logging, time, json, sys
import mwparserfromhell, datetime, math
from pywikibot.data import api
from pywikibot import pagegenerators
from util import config_funcs


#get values from CENSUS API.  Return response from first year with valid response starting with
#current year and ending with 2013
def get_census_values(api_url, get_var, for_var, api_key, year=datetime.datetime.today().year):
    try:
        year = datetime.datetime.today().year
        while year >= 2013:
            payload = {'get': get_var, 'for': for_var, 'key': api_key}
            r = requests.get(api_url.replace('XXXX', str(year)), params=payload)
            if r.status_code == 200:
                return r.json(), str(year)
            else:
                logging.info('No API Results for year: {}'.format(year))
                year = year - 1
        else:
            return
    except requests.exceptions.RequestException as e:
        logging.error('General Exception: {}'.format(e))
    except IOError as err:
        logging.error('IOError: {}'.format(err))

def search_for_page_items(template, infobox_keys):
    template_values = {}
    for item, item_keys in infobox_keys.items():
        for key in item_keys:
            if template.has(key):
                template_values[item] = str(template.get(key))
                print('KEY - {} Found'.format(key))
                break
    return template_values
            
#sort items by population (exluding PR and DC)
def population_rank_sort(pop_list):
    non_states = []
    for i, val in enumerate(pop_list):
        #11 and 72 are the FIPS 5-2 codes for PR and DC
        if val[2] in ['11', '72']:
            non_states.append(pop_list.pop(i))
    pop_list = sorted(pop_list, key=lambda x: int(x[1]), reverse=True)
    ordinal = lambda n: "%d%s" % (n,"tsnrhtdd"[(n//10%10!=1)*(n%10<4)*n%10::4]) 
    for i,val in enumerate(pop_list):
        val.append(ordinal(i+1))
    #leaving this commented out for now because we decided to exclude these from
    #state processing
    #pop_list.extend(non_states)
    return pop_list

#remove value from dict and return copy rather than mutated dict
def removekey(d, key):
    r = dict(d)
    del r[key]
    return r

# compare value from page to value from API to check for discrepancies
# *note - this function does not compare references, only values
def compare_page_items(api_values, page_values, year):
    for key, val in page_values.items():
        pos = int(key.split(' - ')[1])
        # extact value and normalize whitespace
        current_val = ' '.join(val.split('=', 1)[1].split())
        api_value = api_values[pos]
        if key == 'total_pop - 1':
            # format value correctly for comparison with page content
            api_value += ' ('+year+' est)'
            # remove reference, commas, and any <br> tags
            current_val = current_val[:current_val.find('<ref')].replace(',', '').replace('<br>', ' ')
        print('KEY: {} | EXISTING VALUE: {} | NEW VALUE: {}'.format(key.split(' - ')[0], current_val, api_value))
        if current_val == api_value:
            print('VALUES MATCH')
            page_values = removekey(page_values, key)
        else:
            print('VALUES DO NOT MATCH')
    return page_values

def update_page_items(page, text, api_values, page_values, year, reference, comment):
    for key, val in page_values.items(): 
        pos = int(key.split(' - ')[1])
        new_value = api_values[pos]
        # add reference for total population
        if key == 'total_pop - 1':
            new_value = '{:,}'.format(int(new_value))+' ('+year+' est)'+reference
        else:
            index = comment.find('with')
            comment = comment[:index]+' and associated population rank '+comment[index:]
        # add property tag
        new_value = ' '.join(val.split('=', 1)[0].split())+' = '+new_value
        # add new line tag
        new_value += '\n'
        print('FULL EXISTING VALUE: {}\nFULL REPLACEMENT VALUE: {}'.format(val, new_value))
        if not args.debug:
            text = text.replace(val, new_value)
            page.text = text
            page.save(comment)
            print('Page Successfully Updated')
        else:
            print('DEBUG - Page value will be updated')

if __name__ == '__main__':
    scriptpath = os.path.dirname(os.path.abspath(__file__))
   
    parser = argparse.ArgumentParser()
    parser.add_argument(
                        '-m',
                        '--mode',
                        required=True,
                        type=str,
                        choices=['t', 'p'],
                        help='Pass a t flag for test mode or a p flag for production mode'
    )
    parser.add_argument(
                        '-d',
                        '--debug',
                        required=False,
                        action='store_true',
                        default=False,
                        help='Pass this flag to set mode of bot to debug (means there will be no actual changes to the Wiki)'
    )
    args = parser.parse_args()
    logging.info("-------- [SCRIPT ARGUMENTS] --------")
    if args.mode == 't':
        logging.info('      BOT MODE: TEST')
    elif args.mode == 'p':
        logging.info('      BOT MODE: PROD')
    if args.debug:
        logging.info('      !RUNNING IN DEBUG MODE!')
    logging.info("----------- [JOB START] -----------")

    # can be used later for config file
    #data_file = os.path.join(scriptpath, "data", "data.json")

    get_var = 'GEONAME,POP'
    for_var = 'state:*'
    api_url = 'http://api.census.gov/data/XXXX/pep/population'
    api_key = config_funcs.getAppConfigParam('API', 'key')
    # the keys in this dict represent the 'property - position of corresponding value in api response
    # and the values represent the possible keys in which this property is listed under in the template
    infobox_keys = {'total_pop - 1': ['population_total', '2010Pop', '2000Pop', 'population_estimate'],
            'rank - 3': ['PopRank']
            }
    reference = '<ref name=PopEstUS>{{{{cite web|url=https://www.census.gov/programs-surveys/popest.html|title=Population'\
                        ' and Housing Unit Estimates |date={} |accessdate={}|publisher=[[U.S. Census Bureau]]}}}}</ref>'\
                        .format(datetime.datetime.today().strftime('%B %-d, %Y'), datetime.datetime.today().strftime('%B %-d, %Y'))
    comment = 'Updating population estimate with latest data from Census Bureau'
    # position of state code in API response
    code_check_pos = 2
    #DC and PR
    exceptions = ['11', '72']
    key_exceptions = {'Kansas': 'Kansas, United States', 'North Carolina': 'North Carolina, United States',
            'Georgia': 'Georgia, United States', 'Washington': 'Washington (state)'}
    test_data = [['User:Sasan-CDS/sandbox', '555555', '50', '11th']]
    site = pywikibot.Site('en', 'wikipedia') 
    repo = site.data_repository()
    
    if args.mode == 'p':
        metric_values, year = get_census_values(api_url, get_var, for_var, api_key)
        #remove header
        metric_values.pop(0)
        metric_values = population_rank_sort(metric_values)
    else:
        metric_values = test_data
        year = 2016
    if metric_values:
        print('Number of items in API Response: {}'.format(len(metric_values)))
        for i, api_val in enumerate(metric_values):
            key = api_val[0].split(',')[0]
            print('[STATE: {}]'.format(key))
            if key in key_exceptions:
                key = key_exceptions[key]
                print('Key exception found. new key: {}'.format(key))
            if api_val[code_check_pos] in exceptions:
                metric_values.pop(i)
            page = pywikibot.Page(site, key)
            if page.exists():
                if page.isRedirectPage():
                    page = page.getRedirectTarget()
                text = page.get(get_redirect=True)
                code = mwparserfromhell.parse(text)
                template_values = {}
                for template in code.filter_templates():
                    # if correct template is found, break out of loop
                    if template_values:
                        break
                    else:
                        template_values = search_for_page_items(template, infobox_keys)
                if template_values:
                    #compare page items
                    template_values = compare_page_items(api_val, template_values, year)
                    if template_values:
                        update_page_items(page, text, api_val, template_values, year, reference, comment)
                    else:
                        print('Nothing to update')
                else:
                    print('No items were found in this page!!!')
            else:
                print('NO RESULTS FOR: {}'.format(key))
    else:
        sys.exit('NO RESULTS FROM THE CENSUS API FOR ANY YEARS.  EXAMINE FOR OTHER ISSUES!')
<commit_msg>changed test year to string instead of int<commit_after>#!pi_values/usr/bin/python3.5
 
#Author:        <NAME>
#Date:          5/1/17
#Organization:  Commerce Data Service
#Description:   This is a bot script for getting Census data from the Census
#Bureau API's and writing it to Wikipedia.

import pywikibot, json, os, requests, argparse, logging, time, json, sys
import mwparserfromhell, datetime, math
from pywikibot.data import api
from pywikibot import pagegenerators
from util import config_funcs


#get values from CENSUS API.  Return response from first year with valid response starting with
#current year and ending with 2013
def get_census_values(api_url, get_var, for_var, api_key, year=datetime.datetime.today().year):
    try:
        year = datetime.datetime.today().year
        while year >= 2013:
            payload = {'get': get_var, 'for': for_var, 'key': api_key}
            r = requests.get(api_url.replace('XXXX', str(year)), params=payload)
            if r.status_code == 200:
                return r.json(), str(year)
            else:
                logging.info('No API Results for year: {}'.format(year))
                year = year - 1
        else:
            return
    except requests.exceptions.RequestException as e:
        logging.error('General Exception: {}'.format(e))
    except IOError as err:
        logging.error('IOError: {}'.format(err))

def search_for_page_items(template, infobox_keys):
    template_values = {}
    for item, item_keys in infobox_keys.items():
        for key in item_keys:
            if template.has(key):
                template_values[item] = str(template.get(key))
                print('KEY - {} Found'.format(key))
                break
    return template_values
            
#sort items by population (exluding PR and DC)
def population_rank_sort(pop_list):
    non_states = []
    for i, val in enumerate(pop_list):
        #11 and 72 are the FIPS 5-2 codes for PR and DC
        if val[2] in ['11', '72']:
            non_states.append(pop_list.pop(i))
    pop_list = sorted(pop_list, key=lambda x: int(x[1]), reverse=True)
    ordinal = lambda n: "%d%s" % (n,"tsnrhtdd"[(n//10%10!=1)*(n%10<4)*n%10::4]) 
    for i,val in enumerate(pop_list):
        val.append(ordinal(i+1))
    #leaving this commented out for now because we decided to exclude these from
    #state processing
    #pop_list.extend(non_states)
    return pop_list

#remove value from dict and return copy rather than mutated dict
def removekey(d, key):
    r = dict(d)
    del r[key]
    return r

# compare value from page to value from API to check for discrepancies
# *note - this function does not compare references, only values
def compare_page_items(api_values, page_values, year):
    for key, val in page_values.items():
        pos = int(key.split(' - ')[1])
        # extact value and normalize whitespace
        current_val = ' '.join(val.split('=', 1)[1].split())
        api_value = api_values[pos]
        if key == 'total_pop - 1':
            # format value correctly for comparison with page content
            api_value += ' ('+year+' est)'
            # remove reference, commas, and any <br> tags
            current_val = current_val[:current_val.find('<ref')].replace(',', '').replace('<br>', ' ')
        print('KEY: {} | EXISTING VALUE: {} | NEW VALUE: {}'.format(key.split(' - ')[0], current_val, api_value))
        if current_val == api_value:
            print('VALUES MATCH')
            page_values = removekey(page_values, key)
        else:
            print('VALUES DO NOT MATCH')
    return page_values

def update_page_items(page, text, api_values, page_values, year, reference, comment):
    for key, val in page_values.items(): 
        pos = int(key.split(' - ')[1])
        new_value = api_values[pos]
        # add reference for total population
        if key == 'total_pop - 1':
            new_value = '{:,}'.format(int(new_value))+' ('+year+' est)'+reference
        else:
            index = comment.find('with')
            comment = comment[:index]+' and associated population rank '+comment[index:]
        # add property tag
        new_value = ' '.join(val.split('=', 1)[0].split())+' = '+new_value
        # add new line tag
        new_value += '\n'
        print('FULL EXISTING VALUE: {}\nFULL REPLACEMENT VALUE: {}'.format(val, new_value))
        if not args.debug:
            text = text.replace(val, new_value)
            page.text = text
            page.save(comment)
            print('Page Successfully Updated')
        else:
            print('DEBUG - Page value will be updated')

if __name__ == '__main__':
    scriptpath = os.path.dirname(os.path.abspath(__file__))
   
    parser = argparse.ArgumentParser()
    parser.add_argument(
                        '-m',
                        '--mode',
                        required=True,
                        type=str,
                        choices=['t', 'p'],
                        help='Pass a t flag for test mode or a p flag for production mode'
    )
    parser.add_argument(
                        '-d',
                        '--debug',
                        required=False,
                        action='store_true',
                        default=False,
                        help='Pass this flag to set mode of bot to debug (means there will be no actual changes to the Wiki)'
    )
    args = parser.parse_args()
    logging.info("-------- [SCRIPT ARGUMENTS] --------")
    if args.mode == 't':
        logging.info('      BOT MODE: TEST')
    elif args.mode == 'p':
        logging.info('      BOT MODE: PROD')
    if args.debug:
        logging.info('      !RUNNING IN DEBUG MODE!')
    logging.info("----------- [JOB START] -----------")

    # can be used later for config file
    #data_file = os.path.join(scriptpath, "data", "data.json")

    get_var = 'GEONAME,POP'
    for_var = 'state:*'
    api_url = 'http://api.census.gov/data/XXXX/pep/population'
    api_key = config_funcs.getAppConfigParam('API', 'key')
    # the keys in this dict represent the 'property - position of corresponding value in api response
    # and the values represent the possible keys in which this property is listed under in the template
    infobox_keys = {'total_pop - 1': ['population_total', '2010Pop', '2000Pop', 'population_estimate'],
            'rank - 3': ['PopRank']
            }
    reference = '<ref name=PopEstUS>{{{{cite web|url=https://www.census.gov/programs-surveys/popest.html|title=Population'\
                        ' and Housing Unit Estimates |date={} |accessdate={}|publisher=[[U.S. Census Bureau]]}}}}</ref>'\
                        .format(datetime.datetime.today().strftime('%B %-d, %Y'), datetime.datetime.today().strftime('%B %-d, %Y'))
    comment = 'Updating population estimate with latest data from Census Bureau'
    # position of state code in API response
    code_check_pos = 2
    #DC and PR
    exceptions = ['11', '72']
    key_exceptions = {'Kansas': 'Kansas, United States', 'North Carolina': 'North Carolina, United States',
            'Georgia': 'Georgia, United States', 'Washington': 'Washington (state)'}
    test_data = [['User:Sasan-CDS/sandbox', '555555', '50', '11th']]
    site = pywikibot.Site('en', 'wikipedia') 
    repo = site.data_repository()
    
    if args.mode == 'p':
        metric_values, year = get_census_values(api_url, get_var, for_var, api_key)
        #remove header
        metric_values.pop(0)
        metric_values = population_rank_sort(metric_values)
    else:
        metric_values = test_data
        year = '2016'
    if metric_values:
        print('Number of items in API Response: {}'.format(len(metric_values)))
        for i, api_val in enumerate(metric_values):
            key = api_val[0].split(',')[0]
            print('[STATE: {}]'.format(key))
            if key in key_exceptions:
                key = key_exceptions[key]
                print('Key exception found. new key: {}'.format(key))
            if api_val[code_check_pos] in exceptions:
                metric_values.pop(i)
            page = pywikibot.Page(site, key)
            if page.exists():
                if page.isRedirectPage():
                    page = page.getRedirectTarget()
                text = page.get(get_redirect=True)
                code = mwparserfromhell.parse(text)
                template_values = {}
                for template in code.filter_templates():
                    # if correct template is found, break out of loop
                    if template_values:
                        break
                    else:
                        template_values = search_for_page_items(template, infobox_keys)
                if template_values:
                    #compare page items
                    template_values = compare_page_items(api_val, template_values, year)
                    if template_values:
                        update_page_items(page, text, api_val, template_values, year, reference, comment)
                    else:
                        print('Nothing to update')
                else:
                    print('No items were found in this page!!!')
            else:
                print('NO RESULTS FOR: {}'.format(key))
    else:
        sys.exit('NO RESULTS FROM THE CENSUS API FOR ANY YEARS.  EXAMINE FOR OTHER ISSUES!')

{'fraction_non_alphanumeric': 0.1210658945901024, 'fraction_numerical': 0.008900327060211249, 'mean_word_length': 2.1474856564292946, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1728224', 'n_tokens_mistral': 5545, 'n_tokens_neox': 5271, 'n_words': 1539}