<commit_before>#!/usr/bin/python
'''
Created on Dec 14, 2009

@author: edwardsb

This program generates the Burp report, in the format that Vicky requested.

[We would like...] a table that was organized with all the data in one spreadsheet, for example:
              PUBLISHER           PUBLISHER           PUBLISHER
YEAR      # articles ingested # articles ingested # articles ingested
YEAR      # articles ingested # articles ingested # articles ingested
YEAR     # articles ingested # articles ingested # articles ingested 
'''

from datetime import date
import MySQLdb
import optparse
import os
import sys

# Constants
OPTION_BURP_VERSION             = '0.2.1'

OPTION_SHORT                    = '-'
OPTION_LONG                     = '--'

OPTION_DATABASE_PASSWORD        = '<PASSWORD>'
OPTION_DATABASE_PASSWORD_SHORT  = 'D'

OPTION_YEAR_CURRENT             = 'currentyear'
OPTION_YEAR_CURRENT_SHORT       = 'c'
DEFAULT_YEAR_CURRENT            = date.today().year

OPTION_YEAR_MINIMUM             = 'minimumyear'
OPTION_YEAR_MINIMUM_SHORT       = 'm'
DEFAULT_YEAR_MINIMUM            = 1883

OPTION_DATE_REPORT              = 'reportdatestart'
OPTION_DATE_REPORT_SHORT        = 'r'
DEFAULT_DATE_REPORT             = date.today()

OPTION_DATE_REPORT_END          = 'reportdateend'
OPTION_DATE_REPORT_END_SHORT    = 'e'
DEFAULT_DATE_REPORT_END         = date.today()

OPTION_FILENAME_SHORT           = 'f'
OPTION_FILENAME                 = 'filename'
DEFAULT_FILENAME                = 'current.csv'

OPTION_SUMMARY_SHORT            = 's'
OPTION_SUMMARY                  = 'summary'
DEFAULT_SUMMARY                 = 'summary.csv'

def _make_command_line_parser():
    from optparse import OptionGroup, OptionParser

    parser = OptionParser(version=OPTION_BURP_VERSION)

    # The database password
    parser.add_option(OPTION_SHORT + OPTION_DATABASE_PASSWORD_SHORT,
                      OPTION_LONG + OPTION_DATABASE_PASSWORD,
                      dest=OPTION_DATABASE_PASSWORD,
                      help="The password for the database (required)")
    
    # Minimum and current years
    parser.add_option(OPTION_SHORT + OPTION_YEAR_CURRENT_SHORT,
                      OPTION_LONG + OPTION_YEAR_CURRENT,
                      dest=OPTION_YEAR_CURRENT,
                      default=DEFAULT_YEAR_CURRENT,
                      help="The current year (default: %default)")
    parser.add_option(OPTION_SHORT + OPTION_YEAR_MINIMUM_SHORT,
                      OPTION_LONG + OPTION_YEAR_MINIMUM,
                      dest=OPTION_YEAR_MINIMUM,
                      default=DEFAULT_YEAR_MINIMUM,
                      help="The first year to report on (default: %default)")

    # The date for the report: the date that the 'burp.py' program was run.
    parser.add_option(OPTION_SHORT + OPTION_DATE_REPORT_SHORT,
                      OPTION_LONG + OPTION_DATE_REPORT,
                      dest=OPTION_DATE_REPORT,
                      default=DEFAULT_DATE_REPORT,
                      help="The start date the information was gathered (default: %default)")

    parser.add_option(OPTION_SHORT + OPTION_DATE_REPORT_END_SHORT,
                      OPTION_LONG + OPTION_DATE_REPORT_END,
                      dest=OPTION_DATE_REPORT_END,
                      default=DEFAULT_DATE_REPORT_END,
                      help="The end date the information was gathered (default: %default)")

    parser.add_option(OPTION_SHORT + OPTION_FILENAME_SHORT,
                      OPTION_LONG + OPTION_FILENAME,
                      dest=OPTION_FILENAME,
                      default=DEFAULT_FILENAME,
                      help="The filename for the main report (default: %default)")

    parser.add_option(OPTION_SHORT + OPTION_SUMMARY_SHORT,
                      OPTION_LONG + OPTION_SUMMARY,
                      dest=OPTION_SUMMARY,
                      default=DEFAULT_SUMMARY,
                      help="The filename for the summary (default: %default)")

    return parser


def _check_required_options(parser, options):
    if options.dbpassword is None:
        parser.error('%s%s/%s%s is required' % (OPTION_LONG, OPTION_DATABASE_PASSWORD, OPTION_SHORT, OPTION_DATABASE_PASSWORD_SHORT))

def _print_summary_line(summary, publishername, publisherid, currentyear, pubyear, total):
    summary.write(publishername + "," + publisherid.upper() + ",")
    summary.write(str(pubyear[publisherid][currentyear]) + ",")
    summary.write(str(total[publisherid]) + "\n")

def _main_procedure():
    parser = _make_command_line_parser()
    (options, args) = parser.parse_args(values=parser.get_default_values())
    _check_required_options(parser, options)

    # Initialize the hashes.
    # WARNING: If you update this list, you need to update the
    # summary report as well.
    publishers = ['aap', 'aip', 'ama', 'aps', 'acm', 'bep', 'bmc', 'eup', 'elsevier', 'iop', 'wiley', 'lup', 'npg', 'oup', 'rsc', 'sage', 'springer', 'tf', 'rup', 'rs']

    total = {}
    pubyear = {}

    for publisher in publishers:
        pubyear[publisher] = {}
        total[publisher] = 0

        for year in range(options.currentyear, options.minimumyear, -1) + ["0"]:
            strYear = str(year)
            pubyear[publisher][strYear] = 0

    db = MySQLdb.connect(host="localhost", user="edwardsb", passwd=options.dbpassword, db="burp")

    cursorAuid = db.cursor()
    cursorAuid.execute("SELECT DISTINCT(auid) from burp WHERE rundate >= DATE('" + str(options.reportdatestart) + "') AND rundate <= DATE('" + str(options.reportdateend) + "') order by auid;")
    
    auid = cursorAuid.fetchone()
    while auid is not None:
        cursorArticles = db.cursor()
        cursorArticles.execute("SELECT MAX(numarticles), auyear FROM burp WHERE auid = \"" + auid[0] + "\" and rundate >= DATE('" + str(options.reportdatestart) + "') AND rundate <= DATE('" + str(options.reportdateend) +"');")
        articles = cursorArticles.fetchone()
        strYear = articles[1]
        if "-" in strYear:
            strYear = strYear[5:9]
        
        if (auid[0].find("ClockssHighWirePlugin") != -1) and (auid[0].find("aappublications") != -1):
            pubyear['aap'][strYear] += articles[0]
            total['aap'] += articles[0]
            
        if (auid[0].find("ClockssHighWirePlugin") != -1) and (auid[0].find("ama-assn") != -1):
            pubyear['ama'][strYear] += articles[0]
            total['ama'] += articles[0]
            
        if (auid[0].find("ClockssHighWirePlugin") != -1) and (auid[0].find("physiology%2Eorg") != -1):
            pubyear['aps'][strYear] += articles[0]<commit_msg>In Python, the second parameter of 'range' is not inclusive.  In other words, range(1, 5, 2) gives [1, 3] instead of [1, 3, 5].<commit_after>#!/usr/bin/python
'''
Created on Dec 14, 2009

@author: edwardsb

This program generates the Burp report, in the format that Vicky requested.

[We would like...] a table that was organized with all the data in one spreadsheet, for example:
              PUBLISHER           PUBLISHER           PUBLISHER
YEAR      # articles ingested # articles ingested # articles ingested
YEAR      # articles ingested # articles ingested # articles ingested
YEAR     # articles ingested # articles ingested # articles ingested 
'''

from datetime import date
import MySQLdb
import optparse
import os
import sys

# Constants
OPTION_BURP_VERSION             = '0.2.1'

OPTION_SHORT                    = '-'
OPTION_LONG                     = '--'

OPTION_DATABASE_PASSWORD        = '<PASSWORD>'
OPTION_DATABASE_PASSWORD_SHORT  = 'D'

OPTION_YEAR_CURRENT             = 'currentyear'
OPTION_YEAR_CURRENT_SHORT       = 'c'
DEFAULT_YEAR_CURRENT            = date.today().year

OPTION_YEAR_MINIMUM             = 'minimumyear'
OPTION_YEAR_MINIMUM_SHORT       = 'm'
DEFAULT_YEAR_MINIMUM            = 1883

OPTION_DATE_REPORT              = 'reportdatestart'
OPTION_DATE_REPORT_SHORT        = 'r'
DEFAULT_DATE_REPORT             = date.today()

OPTION_DATE_REPORT_END          = 'reportdateend'
OPTION_DATE_REPORT_END_SHORT    = 'e'
DEFAULT_DATE_REPORT_END         = date.today()

OPTION_FILENAME_SHORT           = 'f'
OPTION_FILENAME                 = 'filename'
DEFAULT_FILENAME                = 'current.csv'

OPTION_SUMMARY_SHORT            = 's'
OPTION_SUMMARY                  = 'summary'
DEFAULT_SUMMARY                 = 'summary.csv'

def _make_command_line_parser():
    from optparse import OptionGroup, OptionParser

    parser = OptionParser(version=OPTION_BURP_VERSION)

    # The database password
    parser.add_option(OPTION_SHORT + OPTION_DATABASE_PASSWORD_SHORT,
                      OPTION_LONG + OPTION_DATABASE_PASSWORD,
                      dest=OPTION_DATABASE_PASSWORD,
                      help="The password for the database (required)")
    
    # Minimum and current years
    parser.add_option(OPTION_SHORT + OPTION_YEAR_CURRENT_SHORT,
                      OPTION_LONG + OPTION_YEAR_CURRENT,
                      dest=OPTION_YEAR_CURRENT,
                      default=DEFAULT_YEAR_CURRENT,
                      help="The current year (default: %default)")
    parser.add_option(OPTION_SHORT + OPTION_YEAR_MINIMUM_SHORT,
                      OPTION_LONG + OPTION_YEAR_MINIMUM,
                      dest=OPTION_YEAR_MINIMUM,
                      default=DEFAULT_YEAR_MINIMUM,
                      help="The first year to report on (default: %default)")

    # The date for the report: the date that the 'burp.py' program was run.
    parser.add_option(OPTION_SHORT + OPTION_DATE_REPORT_SHORT,
                      OPTION_LONG + OPTION_DATE_REPORT,
                      dest=OPTION_DATE_REPORT,
                      default=DEFAULT_DATE_REPORT,
                      help="The start date the information was gathered (default: %default)")

    parser.add_option(OPTION_SHORT + OPTION_DATE_REPORT_END_SHORT,
                      OPTION_LONG + OPTION_DATE_REPORT_END,
                      dest=OPTION_DATE_REPORT_END,
                      default=DEFAULT_DATE_REPORT_END,
                      help="The end date the information was gathered (default: %default)")

    parser.add_option(OPTION_SHORT + OPTION_FILENAME_SHORT,
                      OPTION_LONG + OPTION_FILENAME,
                      dest=OPTION_FILENAME,
                      default=DEFAULT_FILENAME,
                      help="The filename for the main report (default: %default)")

    parser.add_option(OPTION_SHORT + OPTION_SUMMARY_SHORT,
                      OPTION_LONG + OPTION_SUMMARY,
                      dest=OPTION_SUMMARY,
                      default=DEFAULT_SUMMARY,
                      help="The filename for the summary (default: %default)")

    return parser


def _check_required_options(parser, options):
    if options.dbpassword is None:
        parser.error('%s%s/%s%s is required' % (OPTION_LONG, OPTION_DATABASE_PASSWORD, OPTION_SHORT, OPTION_DATABASE_PASSWORD_SHORT))

def _print_summary_line(summary, publishername, publisherid, currentyear, pubyear, total):
    summary.write(publishername + "," + publisherid.upper() + ",")
    summary.write(str(pubyear[publisherid][currentyear]) + ",")
    summary.write(str(total[publisherid]) + "\n")

def _main_procedure():
    parser = _make_command_line_parser()
    (options, args) = parser.parse_args(values=parser.get_default_values())
    _check_required_options(parser, options)

    # Initialize the hashes.
    # WARNING: If you update this list, you need to update the
    # summary report as well.
    publishers = ['aap', 'aip', 'ama', 'aps', 'acm', 'bep', 'bmc', 'eup', 'elsevier', 'iop', 'wiley', 'lup', 'npg', 'oup', 'rsc', 'sage', 'springer', 'tf', 'rup', 'rs']

    total = {}
    pubyear = {}

    for publisher in publishers:
        pubyear[publisher] = {}
        total[publisher] = 0

        for year in range(options.currentyear, options.minimumyear - 1, -1) + ["0"]:
            strYear = str(year)
            pubyear[publisher][strYear] = 0

    db = MySQLdb.connect(host="localhost", user="edwardsb", passwd=options.dbpassword, db="burp")

    cursorAuid = db.cursor()
    cursorAuid.execute("SELECT DISTINCT(auid) from burp WHERE rundate >= DATE('" + str(options.reportdatestart) + "') AND rundate <= DATE('" + str(options.reportdateend) + "') order by auid;")
    
    auid = cursorAuid.fetchone()
    while auid is not None:
        cursorArticles = db.cursor()
        cursorArticles.execute("SELECT MAX(numarticles), auyear FROM burp WHERE auid = \"" + auid[0] + "\" and rundate >= DATE('" + str(options.reportdatestart) + "') AND rundate <= DATE('" + str(options.reportdateend) +"');")
        articles = cursorArticles.fetchone()
        strYear = articles[1]
        if "-" in strYear:
            strYear = strYear[5:9]
        
        if (auid[0].find("ClockssHighWirePlugin") != -1) and (auid[0].find("aappublications") != -1):
            pubyear['aap'][strYear] += articles[0]
            total['aap'] += articles[0]
            
        if (auid[0].find("ClockssHighWirePlugin") != -1) and (auid[0].find("ama-assn") != -1):
            pubyear['ama'][strYear] += articles[0]
            total['ama'] += articles[0]
            
        if (auid[0].find("ClockssHighWirePlugin") != -1) and (auid[0].find("physiology%2Eorg") != -1):
            pubyear['aps'][strYear] += articles[0]
{'fraction_non_alphanumeric': 0.10591059105910591, 'fraction_numerical': 0.0065256525652565255, 'mean_word_length': 2.479384133611691, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3190624', 'n_tokens_mistral': 4189, 'n_tokens_neox': 4010, 'n_words': 863}