<commit_before>import argparse
import os

#=============FUNCTION DEFINITIONS===============#
def hline():
    '''This function simply prints a line of 60 "-" chars, which is referenced in many places'''
    print "-" * 60
    
    
def parse_vars(file):
    """This function takes the file with all the variable replacements and puts it into a data structure
        that can later be used to write out each config script.  It also counts the number and tracks 
        the name of hosts found in the file and returns those for quick review prior to exporting the 
        configurations.
        
        The data structure is a dictionary with each hostname as the key, and the value being another 
        dictionary (sub-dictionary?) with each find/replace key and its associated value)"""
    #Reset file position to beginnig (just in case)
    file.seek(0)
    host_list = []
    dict_vars = {}
    host_count = 0
    hostfound = 0
    for line in file:
        if "<--END-->" in line:
            #This marker should be at the end of each host's block of variables, so I know which variables
            #belong to which host in the data structure
            hostfound = 0
        elif "::" not in line:
            #This string "::" is used to separate the key and value in the variable file.  If any line doesn't
            #have this (with exception of the END line, shown above), ignore it and print an error.
            print "Seperator not found in '%r'. Line ignored" % line
        elif "^HOSTNAME^" in line:
            #The Hostname key should always be the first in a block of variables for a device's config file.
            #This name is then used to set the primary key for the device, as well as create the sub-dict
            #which contains the key to match in the template (^HOSTNAME^) and the value to replace it with.
            label, hostname = line.split('::')
            hostname = hostname.rstrip()
            hostfound = 1
            host_count = host_count + 1
            host_list.append(hostname)
            dict_vars[hostname] = {}
            dict_vars[hostname][label] = hostname
        elif hostfound == 1:
            #This should match all find/replace lines under a particular hostname.  This code just adds
            #the pairs to the sub-dictionary for the associated host.
            key, value = line.split('::')
            value = value.rstrip()
            dict_vars[hostname][key] = value
        else:
            #If this section matches a line, something was wrong with the file.  It prints an output of
            #the line and under which host this occurred to help track down the error (for example, it
            #could be HOSTNAME line that had an error, so all lines for that host will match this)
            print "INVALID LINE in %s: '%r'" % (hostname, line)
    #Return the list of hosts, the host count, and the data structure to the main program.
    return host_list, host_count, dict_vars
    
    
def write_configs(info_import, template, verbose):
    """This template writes each host's configuration file.  It should be passed both the template file
        to use, as well as the data structure generated by the parse_vars() function."""
    if verbose:
        hline()
    for host in info_import:
        #reset the template read position to start for each host
        template.seek(0)
        #create file name based on hostname of the device
        filename = host + ".txt"
        hostinfo = info_import[host]
        #Check that the "configs" directory exists.  If not, create it.
        if not os.path.exists("configs"):
            os.makedirs("configs")
        dstfile = open("./configs/" + filename, 'w')
        if verbose:
            print "Starting write of file %s." % filename
            hline()
        for line in template:
            # For each line of the template, do a search for each find/replace "key".  If it is found
            # replace it with the actual value.  Each line is process for every find/replace key in case
            # the line has more than one.  i.e.  ip address ^INSIDE_IP^ ^INSIDE_MASK^
            for key, value in hostinfo.iteritems():
                if key in line:
                    line = line.replace(key,value)
                    if verbose:
                        print "Found an instance of %s and replaced it with '%s'" % (key, value)
            else:
                #After the line has been checked/modified for all applicable keys, write the line to the
                #output file
                dstfile.write(line)
        if verbose:
            print "Configuration %s has been completed" % filename
            hline()
    else:
        #Once the entire loop has finished, all lines in the template have been modified and written.  
        #Close the file.
        print "Configuration files exported."
        dstfile.close()
        
                
def varprint(info):
    """This should be passed the data structure with all the hostname find/replace variables in it.
        This function will crawl the data structure to list all find/replace variables and values
        groups by the host they are meant for."""
    hline()
    for pkey in info:
        print "Variables for device %s:" % pkey
        hline()
        for skey, value in info[pkey].items():
            print "KEY: %s , VALUE: %s" % (skey, value)
        else:
            hline()
            #Pause after each host's output.  Allow a method to quit should the user see some 
            #incorrect data
            pause = raw_input("Press ENTER to Continue (q to QUIT)")
            if pause.lower() == "q":
                print "EXITING DUE TO USER REQUEST"
                exit(0)
            hline()
    else:
        hline()
        print "DONE"
        hline()


def review_vars(data, template, verbose):
    """This is simply a decision tree on whether the user wants to review the variables before
        allowing the program to export all the config files.  It will loop indefinitely until
        a valid respose is received."""
    if verbose:
        varprint(data)
        accept_write(data, template, verbose)
        exit(0)
    while True:
        answ2 = raw_input("Would you like to review the imported data before generating configs? ")
        if answ2.lower() == "yes" or answ2.lower() == "y":
            varprint(data)
            accept_write(data, template, verbose)
            break
        elif answ2.lower() == "no" or answ2.lower() == "n":
            accept_write(data, template, verbose)
            break
        else:
            print "I did not understand that response"
    
    
def accept_write(writedata, template, verbose):
    """This is a decision tree on whether the user wants to export the configuration files."""
    while True:
        answ3 = raw_input("Would you like to export the config files? ")
        if answ3.lower() == "yes" or answ3.lower() == "y":
            write_configs(writedata, template, verbose)
            template.close()
            break
        elif answ3.lower() == "no" or answ3.lower() == "n":
            print "Configuration files have NOT been exported.  Exiting..."
            exit(0)
        else:
            print "I did not understand that answer"
    
    
#=================BEGIN MAIN PROGRAM=====================#

#This section manages the (-h) help output, and makes sure that all required arguments are passed to the script when run
parser = argparse.ArgumentParser(description="This script will generate a configuration file for multiple devices.  It needs a 'template' configuration file that uses variable names in every location where each device needs a different value.  This script also needs a 'variables' file that lists the variables and associated replacement value for the device.  This variables file can contain sections for multiple configuration files.")
group = parser.add_mutually_exclusive_group()
group.add_argument("-v", "--verbose", action="store_true", help="Will provide a more verbose output and automatically trigger a review of all parsed variables")
group.add_argument("-q", "--quiet", action="store_true", help="Will provide minimal output and will automatically skip all review questions")
parser.add_argument("template", help="Name of the file that is the configuration template")
parser.add_argument("variables", help="Name of the file that contains find/replace variables for each device")
#Assign input arguments to the "args" variable for reference later in the script.
args = parser.parse_args()

#Opens the required files for the script to run (template and variables)
var_file = open(args.variables, 'r')
template = open(args.template, 'r')
#Call function to parse variable files and assign outputs to variables
host_list, host_count, importdata = parse_vars(var_file)
#Close variables file -- no longer needed
var_file.close()

#Brief Review of parsed variables
if args.quiet:
    write_configs(importdata, template, args.verbose)
else:
    print "Found settings for %d config files.  They are: " % host_count
    x = 1
    for host in host_list:
        print "%d. %s" % (x, host)
        x = x + 1
    print ""
    #First Decision tree.  Allows user to decide if brief review looks accurate.
    #Other decision tree functions are called depending on the response.
    while True:
        answ = raw_input("Is this correct? (Type 'yes' to continue) ")
        if answ.lower() == "yes" or answ.lower() == "y":
            review_vars(importdata, template, args.verbose)
            break
        elif answ.lower() == "no" or answ.lower() == "n":
            print "Please modify your vars file and try again.  Thanks!"
            template.close()
            exit(0)
        else:
            print "I did not understand that response."
<commit_msg>Added some error checking<commit_after>import argparse
import os
import re

#=============FUNCTION DEFINITIONS===============#
def hline():
    '''This function simply prints a line of 60 "-" chars, which is referenced in many places'''
    print "-" * 60
    
    
def parse_vars(file):
    """This function takes the file with all the variable replacements and puts it into a data structure
        that can later be used to write out each config script.  It also counts the number and tracks 
        the name of hosts found in the file and returns those for quick review prior to exporting the 
        configurations.
        
        The data structure is a dictionary with each hostname as the key, and the value being another 
        dictionary (sub-dictionary?) with each find/replace key and its associated value)"""
    #Reset file position to beginnig (just in case)
    file.seek(0)
    host_list = []
    dict_vars = {}
    host_count = 0
    hostfound = 0
    for line in file:
        if line.startswith("#"):
                pass
        elif "<--END-->" in line:
            #This marker should be at the end of each host's block of variables, so I know which variables
            #belong to which host in the data structure
            hostfound = 0
        elif "::" not in line:
            #This string "::" is used to separate the key and value in the variable file.  If any line doesn't
            #have this (with exception of the END line, shown above), ignore it and print an error.
            print "Did NOT find Seperator '%r'. Line ignored" % line
        elif "<HOSTNAME>" in line and hostfound == 1:
            hline()
            print "ERROR: <HOSTNAME> found inside another host's variable block."
            print "ERROR LINE: '%r'" % line
            print "Exiting program. Please correct line and try again."
            exit(0)
        elif "<HOSTNAME>" in line and hostfound == 0:
            #The Hostname key should always be the first in a block of variables for a device's config file.
            #This name is then used to set the primary key for the device, as well as create the sub-dict
            #which contains the key to match in the template (<HOSTNAME>) and the value to replace it with.
            label, hostname = line.split('::')
            hostname = hostname.rstrip()
            hostfound = 1
            host_count = host_count + 1
            host_list.append(hostname)
            dict_vars[hostname] = {}
            dict_vars[hostname][label] = hostname
        elif hostfound == 1:
            #This should match all find/replace lines under a particular hostname.  This code just adds
            #the pairs to the sub-dictionary for the associated host.
            key, value = line.split('::')
            value = value.rstrip()
            dict_vars[hostname][key] = value
        else:
            #If this section matches a line, something was wrong with the file.  It prints an output of
            #the line and under which host this occurred to help track down the error (for example, it
            #could be HOSTNAME line that had an error, so all lines for that host will match this)
            print "INVALID LINE in %s: '%r'" % (hostname, line)
    #Return the list of hosts, the host count, and the data structure to the main program.
    return host_list, host_count, dict_vars
    
    
def write_configs(info_import, template, verbose):
    """This template writes each host's configuration file.  It should be passed both the template file
        to use, as well as the data structure generated by the parse_vars() function."""
    if verbose:
        hline()
    for host in info_import:
        #reset the template read position to start for each host
        template.seek(0)
        #create file name based on hostname of the device
        filename = host + ".txt"
        hostinfo = info_import[host]
        #Check that the "configs" directory exists.  If not, create it.
        if not os.path.exists("configs"):
            os.makedirs("configs")
        dstfile = open("./configs/" + filename, 'w')
        if verbose:
            print "Starting write of file %s." % filename
            hline()
        for line in template:
            # For each line of the template, do a search for each find/replace "key".  If it is found
            # replace it with the actual value.  Each line is process for every find/replace key in case
            # the line has more than one.  i.e.  ip address <INSIDE_IP> <INSIDE_MASK>
            for key, value in hostinfo.iteritems():
                if key in line:
                    line = line.replace(key,value)
                    if verbose:
                        print "Found an instance of %s and replaced it with '%s'" % (key, value)
            else:
                #After the line has been checked/modified for all applicable keys, write the line to the
                #output file
                dstfile.write(line)
        if verbose:
            print "Configuration %s has been completed" % filename
            hline()
    else:
        #Once the entire loop has finished, all lines in the template have been modified and written.  
        #Close the file.
        print "Configuration files exported."
        dstfile.close()
        
                
def varprint(info):
    """This should be passed the data structure with all the hostname find/replace variables in it.
        This function will crawl the data structure to list all find/replace variables and values
        groups by the host they are meant for."""
    hline()
    for pkey in info:
        print "Variables for device %s:" % pkey
        hline()
        for skey, value in info[pkey].items():
            print "KEY: %s , VALUE: %s" % (skey, value)
        else:
            hline()
            #Pause after each host's output.  Allow a method to quit should the user see some 
            #incorrect data
            pause = raw_input("Press ENTER to Continue (q to QUIT)")
            if pause.lower() == "q":
                print "EXITING DUE TO USER REQUEST"
                exit(0)
            hline()
    else:
        hline()
        print "DONE"
        hline()


def unique_vars(file):
    """This function searches the file for all unique find/replace variables"""
    all_vars = []
    u_var_count = 0

    file.seek(0)
    reg=re.compile(r"<\S*?>")
    for line in file:
        linematch = reg.findall(line)
        for item in linematch:
            if item == "<--END-->":
                pass
            else:
                all_vars.append(item)
    else:
        unique_vars = list(set(all_vars))
    return unique_vars


def var_check(fn_1, fn_2, vars_1, vars_2):
    """This function will take the list of unique variables found in two files and report if they match.
        If they do not match, then this function will output the difference"""
    if len(set(vars_1) & set(vars_2)) == len(set(vars_1)) and len(set(vars_1) & set(vars_2)) == len(set(vars_2)):
        print "All variables match between %s and %s\n" % (fn_1, fn_2)
        return True 
    else:
        print "WARNING:"
        if len(set(vars_1) - set(vars_2)) != 0:
            print "Variables that only exist in %s are: %s\n" % (fn_1, list(set(vars_1) - set(vars_2)))
        if len(set(vars_2) - set(vars_1)) != 0:
            print "Variables that only exist in %s are: %s\n" % (fn_2, list(set(vars_2) - set(vars_1)))
        return False
        

def review_vars(data, template, verbose):
    """This is a decision tree on whether the user wants to review the variables before
        allowing the program to export all the config files.  It will loop indefinitely until
        a valid respose is received."""
    if verbose:
        varprint(data)
        accept_write(data, template, verbose)
        exit(0)
    while True:
        answ2 = raw_input("Would you like to review the imported data before generating configs? ")
        if answ2.lower() == "yes" or answ2.lower() == "y":
            varprint(data)
            accept_write(data, template, verbose)
            break
        elif answ2.lower() == "no" or answ2.lower() == "n":
            accept_write(data, template, verbose)
            break
        else:
            print "I did not understand that response"
    
    
def accept_write(writedata, template, verbose):
    """This is a decision tree on whether the user wants to export the configuration files."""
    while True:
        answ3 = raw_input("Would you like to export the config files? ")
        if answ3.lower() == "yes" or answ3.lower() == "y":
            write_configs(writedata, template, verbose)
            template.close()
            break
        elif answ3.lower() == "no" or answ3.lower() == "n":
            print "Configuration files have NOT been exported.  Exiting..."
            exit(0)
        else:
            print "I did not understand that answer"
    
    
#=================BEGIN MAIN PROGRAM=====================#

#This section manages the (-h) help output, and makes sure that all required arguments are passed to the script when run
parser = argparse.ArgumentParser(description="This script will generate a configuration file for multiple devices.  It needs a 'template' configuration file that uses variable names in every location where each device needs a different value.  This script also needs a 'variables' file that lists the variables and associated replacement value for the device.  This variables file can contain sections for multiple configuration files.")
group = parser.add_mutually_exclusive_group()
group.add_argument("-v", "--verbose", action="store_true", help="Will provide a more verbose output and automatically trigger a review of all parsed variables")
group.add_argument("-q", "--quiet", action="store_true", help="Will provide minimal output and will automatically skip all review questions")
parser.add_argument("template", help="Name of the file that is the configuration template")
parser.add_argument("variables", help="Name of the file that contains find/replace variables for each device")
#Assign input arguments to the "args" variable for reference later in the script.
args = parser.parse_args()

hline()
print""

#Opens the required files for the script to run (template and variables)
var_file = open(args.variables, 'r')
template = open(args.template, 'r')
#Call function to parse variable files and assign outputs to variables
host_list, host_count, importdata = parse_vars(var_file)

var_file_u = unique_vars(var_file)
template_u = unique_vars(template)
correct = var_check(args.variables, args.template, var_file_u, template_u)

#If there were errors in the comparison, allow the user to decide if they want to continue or not.
if not correct:
    while True:
        answ = raw_input("Would you like to continue with the merge anyway? ")
        if answ.lower() == "yes" or answ.lower() == "y":
            break
        elif answ.lower() == "no" or answ.lower() == "n":
            print "Please modify your input files and try again.  Thanks!"
            var_file.close()
            template.close()
            exit(0)
        else:
            print "I did not understand that response."
    

#Close variables file -- no longer needed
var_file.close()

#Brief Review of parsed variables
if args.quiet:
    write_configs(importdata, template, args.verbose)
else:
    print "Found settings for %d config files.  They are: " % host_count
    x = 1
    for host in host_list:
        print "%d. %s" % (x, host)
        x = x + 1
    print ""
    #First Decision tree.  Allows user to decide if brief review looks accurate.
    #Other decision tree functions are called depending on the response.
    while True:
        answ = raw_input("Is this correct? (Type 'yes' to continue) ")
        if answ.lower() == "yes" or answ.lower() == "y":
            review_vars(importdata, template, args.verbose)
            break
        elif answ.lower() == "no" or answ.lower() == "n":
            print "Please modify your vars file and try again.  Thanks!"
            template.close()
            exit(0)
        else:
            print "I did not understand that response."

{'fraction_non_alphanumeric': 0.08760712828186641, 'fraction_numerical': 0.003899696186459892, 'mean_word_length': 2.3496354799513974, 'pattern_counts': {'":': 21, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2385770', 'n_tokens_mistral': 5794, 'n_tokens_neox': 5645, 'n_words': 2587}