<commit_before>"""
Splits into months. Also checks that revisions are from
namespace 0. Note, already filtered upstream in recent changes.

Usage:
    identify_type_of_work_being_done_in_revision (-h|--help)
    identify_type_of_work_being_done_in_revision <input_original> <input_preds> <output>
                                                 [--debug]
                                                 [--verbose]

Options:
    -h, --help        This help message is printed
    <input_original>  Path to original editted file with comments
    <input_preds>     Path to file with ORES and alignment info
    <output>          Where month output will be written
    --debug           Print debug logging to stderr
    --verbose         Print dots and stuff to stderr  
"""


import docopt
import logging
import operator
import mysqltsv
import sys
import re
import json
from collections import defaultdict


logger = logging.getLogger(__name__)


def main(argv=None):
    args = docopt.docopt(__doc__)
    logging.basicConfig(
        level=logging.INFO if not args['--debug'] else logging.DEBUG,
        format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
    )



    input_original_file = mysqltsv.Reader(open(args['<input_original>'],
        'rt', encoding='utf-8', errors='replace'), headers=False,
        types=[int, int, int, str, str, int, int, str, int, int, str, str, str])

    input_preds_file = mysqltsv.Reader(open(args['<input_preds>'],
        'rt', encoding='utf-8', errors='replace'), headers=False,
        types=[str, int, int, str, str, str, str, int, str, str, int, int, int, 
            str, str, str])

    output_file = mysqltsv.Writer(
        open(args['<output>'], "w"), 
        headers=[
                 'page_title',
                 'namespace',
                 'period',
                 'gender',
                 'coordinate_location',
                 'us_location',
                 'edit_type',
                 'rev_id',
                 'expected_quality',
                 'expected_quality_quantile',
                 'page_views',
                 'yyyy',
                 'mm',
                 'does_parent_exist',
                 'parent_weighted_sum',
                 'parent_id',
                 'attribute_modified'])

    verbose = args['--verbose']

    run(input_original_file, input_preds_file, output_file, verbose)


def run(input_original_file, input_preds_file, output_file, verbose):

    revision_comments = defaultdict(str)

    for i, line in enumerate(input_original_file):


        if verbose and i % 10000 == 0 and i != 0:
            sys.stderr.write("Processing original revisions: {0}\n".format(i))  
            sys.stderr.flush()


        if line[7] and re.match(r'/\*\s\S*sitelink', comment, re.I):
            print(re.match(r'/\*\s\S*sitelink', comment, re.I))
        revision_comments[line[6]] = line[7]

    for i, line in enumerate(input_preds_file):

        if verbose and i % 10000 == 0 and i != 0:
            sys.stderr.write("Processing processed revisions: {0}\n".format(i))  
            sys.stderr.flush()

<commit_msg>testing<commit_after>"""
Splits into months. Also checks that revisions are from
namespace 0. Note, already filtered upstream in recent changes.

Usage:
    identify_type_of_work_being_done_in_revision (-h|--help)
    identify_type_of_work_being_done_in_revision <input_original> <input_preds> <output>
                                                 [--debug]
                                                 [--verbose]

Options:
    -h, --help        This help message is printed
    <input_original>  Path to original editted file with comments
    <input_preds>     Path to file with ORES and alignment info
    <output>          Where month output will be written
    --debug           Print debug logging to stderr
    --verbose         Print dots and stuff to stderr  
"""


import docopt
import logging
import operator
import mysqltsv
import sys
import re
import json
from collections import defaultdict


logger = logging.getLogger(__name__)


def main(argv=None):
    args = docopt.docopt(__doc__)
    logging.basicConfig(
        level=logging.INFO if not args['--debug'] else logging.DEBUG,
        format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
    )



    input_original_file = mysqltsv.Reader(open(args['<input_original>'],
        'rt', encoding='utf-8', errors='replace'), headers=False,
        types=[int, int, int, str, str, int, int, str, int, int, str, str, str])

    input_preds_file = mysqltsv.Reader(open(args['<input_preds>'],
        'rt', encoding='utf-8', errors='replace'), headers=False,
        types=[str, int, int, str, str, str, str, int, str, str, int, int, int, 
            str, str, str])

    output_file = mysqltsv.Writer(
        open(args['<output>'], "w"), 
        headers=[
                 'page_title',
                 'namespace',
                 'period',
                 'gender',
                 'coordinate_location',
                 'us_location',
                 'edit_type',
                 'rev_id',
                 'expected_quality',
                 'expected_quality_quantile',
                 'page_views',
                 'yyyy',
                 'mm',
                 'does_parent_exist',
                 'parent_weighted_sum',
                 'parent_id',
                 'attribute_modified'])

    verbose = args['--verbose']

    run(input_original_file, input_preds_file, output_file, verbose)


def run(input_original_file, input_preds_file, output_file, verbose):

    revision_comments = defaultdict(str)

    for i, line in enumerate(input_original_file):


        if verbose and i % 10000 == 0 and i != 0:
            sys.stderr.write("Processing original revisions: {0}\n".format(i))  
            sys.stderr.flush()


        if line[7] and re.match(r'/\*\s\S*sitelink', line[7], re.I):
            print(re.match(r'/\*\s\S*sitelink', line[7], re.I))
        revision_comments[line[6]] = line[7]

    for i, line in enumerate(input_preds_file):

        if verbose and i % 10000 == 0 and i != 0:
            sys.stderr.write("Processing processed revisions: {0}\n".format(i))  
            sys.stderr.flush()


{'fraction_non_alphanumeric': 0.11698717948717949, 'fraction_numerical': 0.007371794871794872, 'mean_word_length': 2.088075210291935, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '382213', 'n_tokens_mistral': 1867, 'n_tokens_neox': 1761, 'n_words': 464}