<commit_before>#!/usr/bin/env python
import sys
import time
import argparse
import psycopg2 as pg
import row_processor as Processor

def _makeDefValues(keys):
    """Returns a dictionary containing None for all keys."""
    return dict(( (k, None) for k in keys ))

def _createMogrificationTemplate(keys):
    """Return the template string for mogrification for the given keys."""
    return '(' + ', '.join( [ '%(' + k + ')s' for k in keys ] ) + ')'

def _createCmdTuple(cursor, keys, templ, attribs):
    """Use the cursor to mogrify a tuple of data.
    The passed data in `attribs` is augmented with default data (NULLs) and the
    order of data in the tuple is the same as in the list of `keys`. The
    `cursor` is used toe mogrify the data and the `templ` is the template used
    for the mogrification.
    """
    defs = _makeDefValues(keys)
    defs.update(attribs)
    return cursor.mogrify(templ, defs)

def handleTable(table, keys, dbname, mbDbFile, mbHost, mbPort, mbUsername, mbPassword):
    """Handle the table including the post/pre processing."""
    dbFile     = mbDbFile if mbDbFile is not None else table + '.xml'
    tmpl       = _createMogrificationTemplate(keys)
    start_time = time.time()

    try:
        pre    = file('./sql/' + table + '_pre.sql').read()
        post   = file('./sql/' + table + '_post.sql').read()<commit_msg>Add special rules section for dealing with mal-formed data.<commit_after>#!/usr/bin/env python
import sys
import time
import argparse
import psycopg2 as pg
import row_processor as Processor

# Special rules needed for certain tables (esp. for old database dumps)
specialRules = {
    ('Posts', 'ViewCount'): "NULLIF(%(ViewCount)s, '')::int"
}

def _makeDefValues(keys):
    """Returns a dictionary containing None for all keys."""
    return dict(( (k, None) for k in keys ))

def _createMogrificationTemplate(table, keys):
    """Return the template string for mogrification for the given keys."""
    return ( '(' +
             ', '.join( [ '%(' + k + ')s' if (table, k) not in specialRules else specialRules[table, k]
                          for k in keys
                        ]
                      ) +
             ')'
           )

def _createCmdTuple(cursor, keys, templ, attribs):
    """Use the cursor to mogrify a tuple of data.
    The passed data in `attribs` is augmented with default data (NULLs) and the
    order of data in the tuple is the same as in the list of `keys`. The
    `cursor` is used toe mogrify the data and the `templ` is the template used
    for the mogrification.
    """
    defs = _makeDefValues(keys)
    defs.update(attribs)
    return cursor.mogrify(templ, defs)

def handleTable(table, keys, dbname, mbDbFile, mbHost, mbPort, mbUsername, mbPassword):
    """Handle the table including the post/pre processing."""
    dbFile     = mbDbFile if mbDbFile is not None else table + '.xml'
    tmpl       = _createMogrificationTemplate(table, keys)
    start_time = time.time()

    try:
        pre    = file('./sql/' + table + '_pre.sql').read()
        post   = file('./sql/' + table + '_post.sql').read()
{'fraction_non_alphanumeric': 0.11422136422136422, 'fraction_numerical': 0.0006435006435006435, 'mean_word_length': 2.9404309252217997, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6461871', 'n_tokens_mistral': 937, 'n_tokens_neox': 923, 'n_words': 352}