<commit_before>import re
import argparse
from multiprocessing import Pool
import traceback
from rdftools.util import log_time
import sys
import os

__author__ = 'basca'

number_pattern = re.compile('"-?(0\.\d*[1-9]\d*|[1-9]\d*(\.\d+)?)"')

#-----------------------------------------------------------------------------------------------------------------------
#
# parallel LUBM generation
#
#-----------------------------------------------------------------------------------------------------------------------
def _round_ntfile(ntfile, round_ntfile, pattern):
    def repl(val):
        return pattern%float(val.group(0)[1:-1])

    try:
        print 'rounding "%s" -> "%s"'%(ntfile, round_ntfile)
        with open(ntfile, 'r+') as NTFILE:
            with open(round_ntfile, 'w+') as ROUND_NTFILE:
                for line in NTFILE:
                    try:
                        r_line = re.sub(number_pattern, repl, line)
                    except Exception, e:
                        print line
                        raise e
                    ROUND_NTFILE.write(r_line)

    except Exception:
        print '[fail]'
        print traceback.format_exc()
    else:
        print '[ok]'

@log_time(None)
def main():
    """
usage: dbpedia_round.py [-h] [--prefix PREFIX] [--precision PRECISION] PATH

rounds ntriple files in a folder, (rounds the floating point literals)

positional arguments:
  PATH                  location of the indexes

optional arguments:
  -h, --help            show this help message and exit
  --prefix PREFIX       the prefix used for files that are transformed, cannot
                        be the enpty string!
  --precision PRECISION
                        the precision to round to, if 0, floating point
                        numbers are rounded to floats
    """
    parser = argparse.ArgumentParser(description='rounds ntriple files in a folder, (rounds the floating point literals)')

    parser.add_argument('path', metavar='PATH', type=str,
                       help='location of the indexes')

    parser.add_argument('--prefix', dest='prefix', action='store', type=str, default='rounded',
                       help='the prefix used for files that are transformed, cannot be the enpty string!')
    parser.add_argument('--precision', dest='precision', action='store', type=long, default=0,
                       help='the precision to round to, if 0, floating point numbers are rounded to floats')

    args = parser.parse_args()

    print 'running dbpedia fpoint round tool'
    if not args.prefix:
        print 'prefix cannot be empty!'
        return

    print 'rounding ... '

    pattern = '"%.'+str(args.precision)+'f"' if args.precision > 0 else '"%d"'

    if os.path.isdir(args.path):
        data_files = [ (os.path.join(args.path, dfile), os.path.join(args.path, '%s_%s'%(args.prefix, dfile)))
                       for dfile in os.listdir(args.path)
                       if not dfile.startswith('.') and os.path.splitext(dfile)[1] == ".nt" ]
    else:
        base, name = os.path.split(args.path)<commit_msg>bumped to version 0.2.9<commit_after>import re
import argparse
from multiprocessing import Pool
import traceback
from rdftools.util import log_time
import sys
import os

__author__ = 'basca'

number_pattern = re.compile('"-?(0\.\d*[1-9]\d*|[1-9]\d*(\.\d+)?)"')

#-----------------------------------------------------------------------------------------------------------------------
#
# parallel LUBM generation
#
#-----------------------------------------------------------------------------------------------------------------------
def _round_ntfile(ntfile, round_ntfile, pattern):
    def repl(val):
        return pattern%float(val.group(0)[1:-1])

    try:
        print 'rounding "%s" -> "%s"'%(ntfile, round_ntfile)
        with open(ntfile, 'r+') as NTFILE:
            with open(round_ntfile, 'w+') as ROUND_NTFILE:
                for line in NTFILE:
                    try:
                        r_line = re.sub(number_pattern, repl, line)
                    except Exception, e:
                        print line
                        raise e
                    ROUND_NTFILE.write(r_line)

    except Exception:
        print '[fail]'
        print traceback.format_exc()
    else:
        print '[ok]'

@log_time(None)
def main():
    """
usage: dbpedia_round.py [-h] [--prefix PREFIX] [--precision PRECISION] PATH

rounds ntriple files in a folder, (rounds the floating point literals)

positional arguments:
  PATH                  location of the indexes

optional arguments:
  -h, --help            show this help message and exit
  --prefix PREFIX       the prefix used for files that are transformed, cannot
                        be the enpty string!
  --precision PRECISION
                        the precision to round to, if 0, floating point
                        numbers are rounded to floats
    """
    parser = argparse.ArgumentParser(description='rounds ntriple files in a folder, (rounds the floating point literals)')

    parser.add_argument('path', metavar='PATH', type=str,
                       help='location of the indexes')

    parser.add_argument('--prefix', dest='prefix', action='store', type=str, default='rounded',
                       help='the prefix used for files that are transformed, cannot be the enpty string!')
    parser.add_argument('--precision', dest='precision', action='store', type=long, default=0,
                       help='the precision to round to, if 0, floating point numbers are rounded to long')

    args = parser.parse_args()

    print 'running dbpedia fpoint round tool'
    if not args.prefix:
        print 'prefix cannot be empty!'
        return

    print 'rounding ... '

    pattern = '"%.'+str(args.precision)+'f"' if args.precision > 0 else '"%d"'

    if os.path.isdir(args.path):
        data_files = [ (os.path.join(args.path, dfile), os.path.join(args.path, '%s_%s'%(args.prefix, dfile)))
                       for dfile in os.listdir(args.path)
                       if not dfile.startswith('.') and os.path.splitext(dfile)[1] == ".nt" ]
    else:
        base, name = os.path.split(args.path)
{'fraction_non_alphanumeric': 0.18610662358642974, 'fraction_numerical': 0.004684975767366721, 'mean_word_length': 1.7345406360424027, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3579863', 'n_tokens_mistral': 1751, 'n_tokens_neox': 1643, 'n_words': 506}