<commit_before>#!/usr/bin/env python2
# This work is licensed under the terms of the MIT license. See LICENSE for the full text.

import os
import sys
import inspect
import math
import cPickle as pickle

path_to_this_module = os.path.abspath( os.path.dirname( inspect.getsourcefile(sys.modules[__name__]) ) )

template_file = os.path.join(path_to_this_module, 'template.py')

def process(data_dict, database_run = False, job_dict = None):
    if database_run:
        required_arguments = [
            'numjobs', 'scriptname', 'mem_free',
            'cluster_rosetta_bin',
            'local_rosetta_bin',
            'output_dir',
            'cluster_rosetta_binary_type',
            'local_rosetta_binary_type',
            'db_id',
        ]
        unrequired_arguments = [
            'add_extra_ld_path',
            'numclusterjobs',
            'appname', 'rosetta_args_list',
            'rosetta_binary_type',
        ]
    else:
        required_arguments = [
            'numjobs', 'scriptname', 'mem_free',
            'cluster_rosetta_bin',
            'local_rosetta_bin',
            'appname', 'rosetta_args_list',
            'output_dir',
            'cluster_rosetta_binary_type',
            'local_rosetta_binary_type',
        ]
        unrequired_arguments = [
            'add_extra_ld_path',
            'numclusterjobs',
            'db_id',
        ]

    for arg in required_arguments:
        if arg not in data_dict:
            print 'ERROR: Data dictionary missing argument', arg
            sys.exit(1)

    for arg in unrequired_arguments:
        if arg in data_dict:
            print 'ERROR: Data dictionary cannot contain argument', arg
            sys.exit(1)

    if database_run:
        data_dict['appname'] = ''
            
    # Handle LD paths
    if 'extra_ld_path' in data_dict:
        data_dict['add_extra_ld_path'] = 'True'
    else:
        data_dict['add_extra_ld_path'] = 'False'
        data_dict['extra_ld_path'] = ''

    data_dict['rosetta_args_list'] = ''
    if not database_run:
        # Handle if general rosetta args are a list instead of a string
        if not isinstance(data_dict['rosetta_args_list'], basestring):
            rosetta_args = ""
            if len(data_dict['rosetta_args_list']) == 1:
                rosetta_args += "'%s'" % data_dict['rosetta_args_list']
            elif len(data_dict['rosetta_args_list']) > 1:
                for arg in data_dict['rosetta_args_list'][:-1]:
                    rosetta_args += "'%s', " % arg
                rosetta_args += "'%s'" % data_dict['rosetta_args_list'][-1]
            data_dict['rosetta_args_list'] = rosetta_args

    # Handle other options
    if 'run_from_database' not in data_dict:
        data_dict['run_from_database'] = False

    if 'db_id' not in data_dict:
        data_dict['db_id'] = ''
    
    if 'cluster_rosetta_db' not in data_dict:
        data_dict['cluster_rosetta_db'] = ''<commit_msg>Fixing bug from plane<commit_after>#!/usr/bin/env python2
# This work is licensed under the terms of the MIT license. See LICENSE for the full text.

import os
import sys
import inspect
import math
import cPickle as pickle

path_to_this_module = os.path.abspath( os.path.dirname( inspect.getsourcefile(sys.modules[__name__]) ) )

template_file = os.path.join(path_to_this_module, 'template.py')

def process(data_dict, database_run = False, job_dict = None):
    if database_run:
        required_arguments = [
            'numjobs', 'scriptname', 'mem_free',
            'cluster_rosetta_bin',
            'local_rosetta_bin',
            'output_dir',
            'cluster_rosetta_binary_type',
            'local_rosetta_binary_type',
            'db_id',
        ]
        unrequired_arguments = [
            'add_extra_ld_path',
            'numclusterjobs',
            'appname', 'rosetta_args_list',
            'rosetta_binary_type',
            'run_from_database',
        ]
    else:
        required_arguments = [
            'numjobs', 'scriptname', 'mem_free',
            'cluster_rosetta_bin',
            'local_rosetta_bin',
            'appname', 'rosetta_args_list',
            'output_dir',
            'cluster_rosetta_binary_type',
            'local_rosetta_binary_type',
        ]
        unrequired_arguments = [
            'add_extra_ld_path',
            'numclusterjobs',
            'run_from_database',
            'db_id',
        ]

    for arg in required_arguments:
        if arg not in data_dict:
            print 'ERROR: Data dictionary missing argument', arg
            sys.exit(1)

    for arg in unrequired_arguments:
        if arg in data_dict:
            print 'ERROR: Data dictionary cannot contain argument', arg
            sys.exit(1)

    if database_run:
        data_dict['appname'] = ''
        data_dict['run_from_database'] = 'True'
    else:
        data_dict['run_from_database'] = 'False'
            
    # Handle LD paths
    if 'extra_ld_path' in data_dict:
        data_dict['add_extra_ld_path'] = 'True'
    else:
        data_dict['add_extra_ld_path'] = 'False'
        data_dict['extra_ld_path'] = ''

    data_dict['rosetta_args_list'] = ''
    if not database_run:
        # Handle if general rosetta args are a list instead of a string
        if not isinstance(data_dict['rosetta_args_list'], basestring):
            rosetta_args = ""
            if len(data_dict['rosetta_args_list']) == 1:
                rosetta_args += "'%s'" % data_dict['rosetta_args_list']
            elif len(data_dict['rosetta_args_list']) > 1:
                for arg in data_dict['rosetta_args_list'][:-1]:
                    rosetta_args += "'%s', " % arg
                rosetta_args += "'%s'" % data_dict['rosetta_args_list'][-1]
            data_dict['rosetta_args_list'] = rosetta_args

    # Handle other options
    if 'db_id' not in data_dict:
        data_dict['db_id'] = ''
    
    if 'cluster_rosetta_db' not in data_dict:
        data_dict['cluster_rosetta_db'] = ''
{'fraction_non_alphanumeric': 0.10414577064526914, 'fraction_numerical': 0.002340354396522902, 'mean_word_length': 2.230561555075594, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1864113', 'n_tokens_mistral': 1910, 'n_tokens_neox': 1872, 'n_words': 389}