<commit_before>#! /usr/bin/env python3

########################################################################
#                                                                      #
# This script was written by <NAME> in 2015.                    #
#        <EMAIL>     <EMAIL>                    #
#                                                                      #
# Copyright 2015 <NAME>                                   #
#                                                                      #
# Licensed under the Apache License, Version 2.0 (the "License");      #
# you may not use this file except in compliance with the License.     #
# You may obtain a copy of the License at                              #
#                                                                      #
#    http://www.apache.org/licenses/LICENSE-2.0                        #
#                                                                      #
# Unless required by applicable law or agreed to in writing, software  #
# distributed under the License is distributed on an "AS IS" BASIS,    #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or      #
# implied.                                                             #
# See the License for the specific language governing permissions and  #
# limitations under the License.                                       #
#                                                                      #
########################################################################

# This is written to work with python 3 because it should be good to
# be working on the newest version of python.
from __future__ import print_function

import argparse   # For parsing commandline arguments
import glob       # Allows referencing file system/file names
import readline   # Allows easier file input (with tab completion?)
import subprocess # Allows for submitting commands to the shell

yes = ['y', 'yes', '1']


# An input function that can prefill in the text entry
# Not sure if this works in 3.5+ because raw_input is gone
def rlinput(prompt, prefill=''):
    readline.set_startup_hook(lambda: readline.insert_text(prefill))
    try:
        return input(prompt)
    finally:
        readline.set_startup_hook()


def create_gau_input(coord_name,template):
    """This function takes as input a file with a set of molecular
    coordinates (the form should not matter, it will just be copied
    into the next file) and a template file that should be the header
    for the desired calculation (including charge and multiplicity),
    returns nothing, but creates a Gaussian input file ending with
    '.com' """
    print('Creating Gaussian input file...')
    _out_name = coord_name.rsplit('.', 1)[0] + '.com'
    with open(_out_name, 'w') as out_file:
        with open(template, 'r') as templ_file:
            if args.verbose:
                print('opened {}'.format(template))
            for line in templ_file:
                out_file.write(line)
            if '\n' not in line:
                out_file.write('\n')
        with open(coord_name, 'r') as in_file:
            if args.verbose:
                print('opened {}'.format(coord_name))
            for i, line in enumerate(in_file):
                if i < 2:
                    # ignore first two lines
                    # number of atoms and the title/comment
                    continue
                # if line.strip().isdigit():
                #     # the first line is the number of atoms
                #     continue
                # # XYZ files created by mathematica have a comment
                # # as the second line saying something like:
                # # "Created by mathematica". Obv. want to ignore that
                # if line.strip().startswith('Create') or
                #         line.strip().startswith('generated'):
                #     continue
                # else:
                out_file.write(line)
        out_file.write('\n\n\n')
    if args.verbose:
        print('created Gaussian input file {}'.format(_out_name))
    return _out_name


def get_input_files(base_name, batch):
    _in_name_list = glob.glob(base_name + '*')
    _in_name_list.sort()        # sort files alphanumerically
    _in_name_list.sort(key=len) # sort by length (because otherwise would
    # put 1,10,11,... as opposed to 1,...,9,10,...
    # if number 01,02,... They should all be the same length and the
    # second sort won't do anything.
    if not batch:
        num_files = len(_in_name_list)
        if num_files > 1:
            print('Multiple files starting with {}'.format(base_name))
            if input('Did you mean to execute a batch job? ') in yes:
                batch = True
            else:
                print('What file name shall I use?')
                _in_name_list = [rlinput('file name: ', base_name)]
    return _in_name_list, batch


def use_template(template, in_names, verbose):
    made_name_list = []
    for in_name in in_names:
        out_name = create_gau_input(in_name, template)
        made_name_list.append(out_name)
        if verbose:
            print('Added {} to files to possibly submit.'.format(out_name))
    _in_name_list = made_name_list
    _in_name_list.sort()
    _in_name_list.sort(key=len)
    return _in_name_list


def write_sub_script(input_name, num_cores, time, verbose):
    if input_name.endswith('.com'):
        short_name = input_name.rsplit('.', 1)[0]
        if not short_name + '.com' == input_name:
            raise SyntaxError('problem interpreting file name. ' +
                              'Period in file name?')
        out_name = short_name + '.out'
    elif '.' in input_name:
        short_name = input_name.rsplit('.', 1)[0]
        input_extension = input_name.rsplit('.', 1)[-1]
        if not short_name + '.' + input_extension == input_name:
            raise SyntaxError('problem interpreting file name. ' +
                              'Period in file name?')
        out_name = short_name + '.out'
    else:
        short_name = input_name
        input_name = short_name + '.com'
        print('Assuming input file is {}'.format(input_name))
        out_name = short_name + '.out'

    _script_name = 'submit' + short_name + '.sh'

    with open(_script_name, 'w') as script_file:
        script_file.write('#!/bin/sh \n\n')
        script_file.write('#$ -pe omp {}\n'.format(num_cores))
        script_file.write('#$ -M <EMAIL>\n')
        script_file.write('#$ -m eas\n')
        script_file.write('#$ -l h_rt={}\n'.format(time))
        script_file.write('#$ -N {}\n'.format(short_name))
        script_file.write('#$ -j y\n')
        script_file.write('#$ -o {}.log\n\n'.format(short_name))
        script_file.write('INPUTFILE={}\n'.format(input_name))
        script_file.write('OUTPUTFILE={}\n\n'.format(out_name))
        script_file.write('CURRENTDIR=`pwd`\n')
        script_file.write('SCRATCHDIR=/scratch/$USER\n')
        script_file.write('mkdir -p $SCRATCHDIR\n\n')
        script_file.write('cd $SCRATCHDIR\n\n')
        script_file.write('cp $CURRENTDIR/$INPUTFILE .\n\n')
        script_file.write('echo About to run g09 in /net/`'
                          'hostname -s`$SCRATCHDIR\n\n')
        script_file.write('g09 <$INPUTFILE > $OUTPUTFILE\n\n')
        script_file.write('cp $OUTPUTFILE $CURRENTDIR/.\n\n')
        script_file.write('echo ran in /net/`hostname -s`$SCRATCHDIR\n')
        script_file.write('echo output was copied to $CURRENTDIR\n\n')

    if verbose:
        print('script written to {}'.format(script_name))
    return _script_name


def submit_scripts(scripts, batch, submit, verbose):
    # todo return formatted job number/name
    if batch:
        if submit or input('submit all jobs? ') in yes:
            for script in scripts:
                cl = 'qsub {}'.format(script)
                # Don't really know how this works. Copied from
                # http://stackoverflow.com/questions/4256107/
                # running-bash-commands-in-python
                process = subprocess.Popen(cl.split(),
                                           stdout=subprocess.PIPE,
                                           universal_newlines=True)
                output = process.communicate()[0]
                print(output)
        else:
            if verbose:
                print('No jobs submitted, but scripts created')
    else:
        if submit or input('submit job {}? '.format(scripts[0])) in yes:<commit_msg>added support for nested files in write_sub_script<commit_after>#! /usr/bin/env python3

########################################################################
#                                                                      #
# This script was written by <NAME> in 2015.                    #
#        <EMAIL>     <EMAIL>                    #
#                                                                      #
# Copyright 2015 <NAME> IV                                   #
#                                                                      #
# Licensed under the Apache License, Version 2.0 (the "License");      #
# you may not use this file except in compliance with the License.     #
# You may obtain a copy of the License at                              #
#                                                                      #
#    http://www.apache.org/licenses/LICENSE-2.0                        #
#                                                                      #
# Unless required by applicable law or agreed to in writing, software  #
# distributed under the License is distributed on an "AS IS" BASIS,    #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or      #
# implied.                                                             #
# See the License for the specific language governing permissions and  #
# limitations under the License.                                       #
#                                                                      #
########################################################################

# This is written to work with python 3 because it should be good to
# be working on the newest version of python.
from __future__ import print_function

import argparse   # For parsing commandline arguments
import glob       # Allows referencing file system/file names
import readline   # Allows easier file input (with tab completion?)
import subprocess # Allows for submitting commands to the shell

yes = ['y', 'yes', '1']


# An input function that can prefill in the text entry
# Not sure if this works in 3.5+ because raw_input is gone
def rlinput(prompt, prefill=''):
    readline.set_startup_hook(lambda: readline.insert_text(prefill))
    try:
        return input(prompt)
    finally:
        readline.set_startup_hook()


def create_gau_input(coord_name,template):
    """This function takes as input a file with a set of molecular
    coordinates (the form should not matter, it will just be copied
    into the next file) and a template file that should be the header
    for the desired calculation (including charge and multiplicity),
    returns nothing, but creates a Gaussian input file ending with
    '.com' """
    print('Creating Gaussian input file...')
    _out_name = coord_name.rsplit('.', 1)[0] + '.com'
    with open(_out_name, 'w') as out_file:
        with open(template, 'r') as templ_file:
            if args.verbose:
                print('opened {}'.format(template))
            for line in templ_file:
                out_file.write(line)
            if '\n' not in line:
                out_file.write('\n')
        with open(coord_name, 'r') as in_file:
            if args.verbose:
                print('opened {}'.format(coord_name))
            for i, line in enumerate(in_file):
                if i < 2:
                    # ignore first two lines
                    # number of atoms and the title/comment
                    continue
                # if line.strip().isdigit():
                #     # the first line is the number of atoms
                #     continue
                # # XYZ files created by mathematica have a comment
                # # as the second line saying something like:
                # # "Created by mathematica". Obv. want to ignore that
                # if line.strip().startswith('Create') or
                #         line.strip().startswith('generated'):
                #     continue
                # else:
                out_file.write(line)
        out_file.write('\n\n\n')
    if args.verbose:
        print('created Gaussian input file {}'.format(_out_name))
    return _out_name


def get_input_files(base_name, batch):
    _in_name_list = glob.glob(base_name + '*')
    _in_name_list.sort()        # sort files alphanumerically
    _in_name_list.sort(key=len) # sort by length (because otherwise would
    # put 1,10,11,... as opposed to 1,...,9,10,...
    # if number 01,02,... They should all be the same length and the
    # second sort won't do anything.
    if not batch:
        num_files = len(_in_name_list)
        if num_files > 1:
            print('Multiple files starting with {}'.format(base_name))
            if input('Did you mean to execute a batch job? ') in yes:
                batch = True
            else:
                print('What file name shall I use?')
                _in_name_list = [rlinput('file name: ', base_name)]
    return _in_name_list, batch


def use_template(template, in_names, verbose):
    made_name_list = []
    for in_name in in_names:
        out_name = create_gau_input(in_name, template)
        made_name_list.append(out_name)
        if verbose:
            print('Added {} to files to possibly submit.'.format(out_name))
    _in_name_list = made_name_list
    _in_name_list.sort()
    _in_name_list.sort(key=len)
    return _in_name_list


def write_sub_script(input_name, num_cores, time, verbose):
    if '/' in input_name:
        rel_dir, file_name = input_name.rsplit('/', 1)
    else:
        rel_dir = './'
        file_name = input_name
    if file_name.endswith('.com'):
        short_name = file_name.rsplit('.', 1)[0]
        if not short_name + '.com' == file_name:
            raise SyntaxError('problem interpreting file name. ' +
                              'Period in file name?')
        out_name = short_name + '.out'
    elif '.' in file_name:
        short_name = file_name.rsplit('.', 1)[0]
        input_extension = file_name.rsplit('.', 1)[-1]
        if not short_name + '.' + input_extension == file_name:
            raise SyntaxError('problem interpreting file name. ' +
                              'Period in file name?')
        out_name = short_name + '.out'
    else:
        short_name = file_name
        file_name = short_name + '.com'
        print('Assuming input file is {}'.format(file_name))
        out_name = short_name + '.out'

    _script_name = rel_dir + 'submit' + short_name + '.sh'

    with open(_script_name, 'w') as script_file:
        script_file.write('#!/bin/sh \n\n')
        script_file.write('#$ -pe omp {}\n'.format(num_cores))
        script_file.write('#$ -M <EMAIL>\n')
        script_file.write('#$ -m eas\n')
        script_file.write('#$ -l h_rt={}\n'.format(time))
        script_file.write('#$ -N {}\n'.format(short_name))
        script_file.write('#$ -j y\n')
        script_file.write('#$ -o {}.log\n\n'.format(short_name))
        script_file.write('INPUTFILE={}\n'.format(input_name))
        script_file.write('OUTPUTFILE={}\n\n'.format(out_name))
        script_file.write('CURRENTDIR=`pwd`\n')
        script_file.write('SCRATCHDIR=/scratch/$USER\n')
        script_file.write('mkdir -p $SCRATCHDIR\n\n')
        script_file.write('cd $SCRATCHDIR\n\n')
        script_file.write('cp $CURRENTDIR/$INPUTFILE .\n\n')
        script_file.write('echo About to run g09 in /net/`'
                          'hostname -s`$SCRATCHDIR\n\n')
        script_file.write('g09 <$INPUTFILE > $OUTPUTFILE\n\n')
        script_file.write('cp $OUTPUTFILE $CURRENTDIR/.\n\n')
        script_file.write('echo ran in /net/`hostname -s`$SCRATCHDIR\n')
        script_file.write('echo output was copied to $CURRENTDIR\n\n')

    if verbose:
        print('script written to {}'.format(_script_name))
    return _script_name


def submit_scripts(scripts, batch, submit, verbose):
    # todo return formatted job number/name
    if batch:
        if submit or input('submit all jobs? ') in yes:
            for script in scripts:
                cl = 'qsub {}'.format(script)
                # Don't really know how this works. Copied from
                # http://stackoverflow.com/questions/4256107/
                # running-bash-commands-in-python
                process = subprocess.Popen(cl.split(),
                                           stdout=subprocess.PIPE,
                                           universal_newlines=True)
                output = process.communicate()[0]
                print(output)
        else:
            if verbose:
                print('No jobs submitted, but scripts created')
    else:
        if submit or input('submit job {}? '.format(scripts[0])) in yes:
{'fraction_non_alphanumeric': 0.11814272431427243, 'fraction_numerical': 0.006218038121803812, 'mean_word_length': 1.792308940451079, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '745359', 'n_tokens_mistral': 4752, 'n_tokens_neox': 4524, 'n_words': 1356}