<commit_before>#! /usr/bin/env python3.4

########################################################################
#                                                                      #
#                                                                      #
#                                                                      #
# Known issues:                                                        #
# Methods which are declared with non-alphanumeric characters          #
#                                                                      #
#                                                                      #
#                                                                      #
# This script was written by <NAME> in 2015.                    #
#        <EMAIL>     <EMAIL>                    #
#                                                                      #
# Copyright 2015 <NAME> IV                                   #      
#                                                                      #
# Licensed under the Apache License, Version 2.0 (the "License");      #
# you may not use this file except in compliance with the License.     #
# You may obtain a copy of the License at                              #
#                                                                      #
#    http://www.apache.org/licenses/LICENSE-2.0                        #
#                                                                      #
# Unless required by applicable law or agreed to in writing, software  #
# distributed under the License is distributed on an "AS IS" BASIS,    #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or      #
# implied.                                                             #
# See the License for the specific language governing permissions and  #
# limitations under the License.                                       #
#                                                                      #
########################################################################




# This is written to work with python 3.4 because it should be good to
# be working on the newest version of python.

import fileinput  # allows easy iteration over a file
import re         # RegEx package for sorting data
import glob       # Allows referencing file system/file names
import argparse   # For parsing commandline arguments

# Really should adapt to use argparse for parsing command line
# arguments. see https://docs.python.org/2/library/argparse.html
# Probably particularly useful for setting a flag for method
# or multistate input files. I can try to detect, and just
# take it as a command line argument for ease. May not be any
# faster as I already iterate through every line of the file.

descrip = ('This function takes a base file name for output files and'+
    ' makes a file basename_energies that is a list of the energies ' +
    'from the the read output files.')

parser = argparse.ArgumentParser(description=descrip)
parser.add_argument('base_name', help='base name of files to read')
parser.add_argument('-m', '--method',
                    help='calculation method (changes file interp')
args = parser.parse_args()

base_name     =     args.base_name
in_name_list  =     glob.glob(base_name + '*.out')
num_files     = len(in_name_list)

in_name_list.sort()        # sort files alphanumerically
in_name_list.sort(key=len) # sort by length (because otherwise would
# out 1,10,11,... as opposed to 1,...,9,10,...
# if number 01,02,... They should all be the same length and the
# second sort won't do anything.

print('base file name is {}'.format(base_name))
print('number of files is {:,}'.format(num_files))
print('Energies of {}'.format(base_name))


out_name = base_name + '_energies'

# Define methods I might use that change how to process file.
methods = {'wb97xd': 'dft', 'mp2': 'mp2', 'pbe50': 'sfdft',
           'eom-ccsd': 'sf', 'hf': 'hf', 'pbe1pbe': 'gdft',
           'cis': 'gcis'}
# reading more about regex, I really should use them. I could
# define how it matches based on the method, as opposed to all
# the If Then loops I currently have.
# I could essentially have a dictionary of regular expressions and just
# reference the appropriate one for the type of output it's reading.
# Alternatively, I could just break it up into a lot of subroutines
# that could be called depending on the type of output.

i = 0
# Initialize not knowing method used: "Not Yet Determined"
method = 'nyd'
# If given as commandline argument, set with the given method
if args.method:
    method = methods[args.method.lower()]

# Gaussian method regex:
# finds alphanumeric character string before a / and after a
# whitespace character. Requires that the input be in the form
# (method)/(something, normally basis set).
# Are there methods/functionals with non-alphanumeric characters?
# Maybe manual hybrid functions would be bad with this. I don't
# currently remember their syntax.
# This has problems with CIS which uses parenthesis in the syntax.
# Currently, just provide it as a command line argument with -m.
gregex = re.compile(r"\s+\w+/")
# QChem SF methods energy regex:
sfenergy = re.compile('[=:]\s*-\d*\.\d*')
# Gaussian DFT methods ground state energy regex:
# (could use sfenergy, or combine into one)
gdftenergy = re.compile(r'=\s*-\d*\.\d*')
# Gaussian TD-DFT methods excited state energies regex:
tdgdftenergy = re.compile(r'\s-*\d+\.\d+\s+ev')
# Gaussian CIS ground state energy regex:
# Goes to end of the line because given in scientific notation.
gcisenergy = re.compile(r'eump2\s*=\s*-.+')

with open(out_name, 'w') as out_file:
    for name in in_name_list:
        energy_list = []
        with open(name, 'r') as in_file:
            for line in in_file:
                line = line.lower().strip()
                if method is 'nyd':
                    if 'method' in line or 'exchange' in line:
                        # make the line into a list split by spaces
                        linelist = re.split(' +',line)
                        # could maybe shorten these next lines with
                        # a Try Except construction
                        if linelist[-1] in methods:
                            method = methods[linelist[-1]]
                        else:
                            print('Unknown method {} used'.format(
                                linelist[-1]))
                            print('Assuming output formatted as HF')
                            method = 'hf'
                    if 'entering gaussian sys' in line:
                        method = 'gaugen'
                        # Gaussian output file, method undetermined
                    continue
                if method is 'gaugen':
                    if line.startswith('#'):
                        gmethodmatch = gregex.search(line)
                        gmethod =re.search(r"\w+", gmethodmatch.group())
                        if gmethod:
                            try:
                                method = methods[gmethod.group()]
                            except KeyError:
                                print('unknown Gaussian method. ' +
                                      'Assuming (g)dft.')
                                method = 'gdft'
                        if re.search(r'\btd\s*[(=]', line):
                            method = 'td' + method
                            # Note, this will cause problems if TD is
                            # declared on a line before the<commit_msg>Should now work with CIS and other Gaussian methods with non-alphanumeric characters before the /.<commit_after>#! /usr/bin/env python3.4

########################################################################
#                                                                      #
#                                                                      #
#                                                                      #
# Known issues:                                                        #
# Methods which are declared with non-alphanumeric characters          #
#                                                                      #
#                                                                      #
#                                                                      #
# This script was written by <NAME> in 2015.                    #
#        <EMAIL>     <EMAIL>                    #
#                                                                      #
# Copyright 2015 <NAME> IV                                   #      
#                                                                      #
# Licensed under the Apache License, Version 2.0 (the "License");      #
# you may not use this file except in compliance with the License.     #
# You may obtain a copy of the License at                              #
#                                                                      #
#    http://www.apache.org/licenses/LICENSE-2.0                        #
#                                                                      #
# Unless required by applicable law or agreed to in writing, software  #
# distributed under the License is distributed on an "AS IS" BASIS,    #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or      #
# implied.                                                             #
# See the License for the specific language governing permissions and  #
# limitations under the License.                                       #
#                                                                      #
########################################################################




# This is written to work with python 3.4 because it should be good to
# be working on the newest version of python.

import fileinput  # allows easy iteration over a file
import re         # RegEx package for sorting data
import glob       # Allows referencing file system/file names
import argparse   # For parsing commandline arguments

# Really should adapt to use argparse for parsing command line
# arguments. see https://docs.python.org/2/library/argparse.html
# Probably particularly useful for setting a flag for method
# or multistate input files. I can try to detect, and just
# take it as a command line argument for ease. May not be any
# faster as I already iterate through every line of the file.

descrip = ('This function takes a base file name for output files and'+
    ' makes a file basename_energies that is a list of the energies ' +
    'from the the read output files.')

parser = argparse.ArgumentParser(description=descrip)
parser.add_argument('base_name', help='base name of files to read')
parser.add_argument('-m', '--method',
                    help='calculation method (changes file interp')
args = parser.parse_args()

base_name     =     args.base_name
in_name_list  =     glob.glob(base_name + '*.out')
num_files     = len(in_name_list)

in_name_list.sort()        # sort files alphanumerically
in_name_list.sort(key=len) # sort by length (because otherwise would
# out 1,10,11,... as opposed to 1,...,9,10,...
# if number 01,02,... They should all be the same length and the
# second sort won't do anything.

print('base file name is {}'.format(base_name))
print('number of files is {:,}'.format(num_files))
print('Energies of {}'.format(base_name))


out_name = base_name + '_energies'

# Define methods I might use that change how to process file.
methods = {'wb97xd': 'dft', 'mp2': 'mp2', 'pbe50': 'sfdft',
           'eom-ccsd': 'sf', 'hf': 'hf', 'pbe1pbe': 'gdft',
           'cis': 'gcis'}
# reading more about regex, I really should use them. I could
# define how it matches based on the method, as opposed to all
# the If Then loops I currently have.
# I could essentially have a dictionary of regular expressions and just
# reference the appropriate one for the type of output it's reading.
# Alternatively, I could just break it up into a lot of subroutines
# that could be called depending on the type of output.

i = 0
# Initialize not knowing method used: "Not Yet Determined"
method = 'nyd'
# If given as commandline argument, set with the given method
if args.method:
    method = methods[args.method.lower()]

# Gaussian method regex:
# finds non-whitespace characters before a / and after a
# whitespace character. Requires that the input be in the form
# (method)/(something, normally basis set).
gregex = re.compile(r"\s+\S+/")
# QChem SF methods energy regex:
sfenergy = re.compile('[=:]\s*-\d*\.\d*')
# Gaussian DFT methods ground state energy regex:
# (could use sfenergy, or combine into one)
gdftenergy = re.compile(r'=\s*-\d*\.\d*')
# Gaussian TD-DFT methods excited state energies regex:
tdgdftenergy = re.compile(r'\s-*\d+\.\d+\s+ev')
# Gaussian CIS ground state energy regex:
# Goes to end of the line because given in scientific notation.
gcisenergy = re.compile(r'eump2\s*=\s*-.+')

with open(out_name, 'w') as out_file:
    for name in in_name_list:
        energy_list = []
        with open(name, 'r') as in_file:
            for line in in_file:
                line = line.lower().strip()
                if method is 'nyd':
                    if 'method' in line or 'exchange' in line:
                        # make the line into a list split by spaces
                        linelist = re.split(' +',line)
                        # could maybe shorten these next lines with
                        # a Try Except construction
                        if linelist[-1] in methods:
                            method = methods[linelist[-1]]
                        else:
                            print('Unknown method {} used'.format(
                                linelist[-1]))
                            print('Assuming output formatted as HF')
                            method = 'hf'
                    if 'entering gaussian sys' in line:
                        method = 'gaugen'
                        # Gaussian output file, method undetermined
                    continue
                if method is 'gaugen':
                    if line.startswith('#'):
                        gmethodmatch = gregex.search(line)
                        if gmethodmatch:
                            gmethod =re.search(r"\w+", gmethodmatch.group())
                        if gmethod:
                            try:
                                method = methods[gmethod.group()]
                            except KeyError:
                                print('unknown Gaussian method. ' +
                                      'Assuming (g)dft.')
                                method = 'gdft'
                        if re.search(r'\btd\s*[(=]', line):
                            method = 'td' + method
                            # Note, this will cause problems if TD is
                            # declared on a line before the
{'fraction_non_alphanumeric': 0.1000335232986926, 'fraction_numerical': 0.005631914180355347, 'mean_word_length': 1.4642326119279696, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 60, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5124643', 'n_tokens_mistral': 3538, 'n_tokens_neox': 3347, 'n_words': 1363}