<commit_before>#!/bin/bash/python

from __future__ import print_function
import dataConsistencyErrors as er
import superpipeline

import json
import os
import sys

# Loop over all of the nodes in a graph and check that the values associated with it
# are of the correct type and have extensions consistent with all of the arguments
# attached to the node.
def checkValues(graph, superpipeline):

  # First, loop over all of the option nodes and check that the data types for all values
  # are valid.
  for nodeID in graph.getNodes(['option', 'file']):
    values   = graph.getGraphNodeAttribute(nodeID, 'values')
    nodeType = graph.getGraphNodeAttribute(nodeID, 'nodeType')

    # Get all of the arguments that use this node and check the data types. Start with all predecessors to this node.
    expectedDataType = None
    for predecessorNodeID in graph.getPredecessors(nodeID):
      expectedDataType = checkNode(graph, superpipeline, predecessorNodeID, nodeID, nodeType, expectedDataType, values, isInput = False)
    for successorNodeID in graph.getSuccessors(nodeID):
      expectedDataType = checkNode(graph, superpipeline, nodeID, successorNodeID, nodeType, expectedDataType, values, isInput = True)

# Check the values for a node.
def checkNode(graph, superpipeline, source, target, nodeType, expectedDataType, values, isInput):
 
  # Define error handling,
  errors = er.consistencyErrors()

  # Get pipeline configuration data.
  data = superpipeline.pipelineConfigurationData[superpipeline.pipeline]

  # Get the required attributes.
  longFormArgument = graph.CM_getArgumentAttribute(graph.graph, source, target, 'longFormArgument')
  dataType         = graph.CM_getArgumentAttribute(graph.graph, source, target, 'dataType')

  # If this is the first argument parsed, populate the expectedDataType variable with the data type for this argument.
  if not expectedDataType: expectedDataType = dataType

  # If expectedDataType is populated and this data type is different to the expectedDataType, this implies that different
  # arguments using the same values expect different data types. This is clearly impossible, so terminate.
  #TODO ERROR
  elif expectedDataType != dataType: print('dataConsistency.checkNode - 1', dataType, expectedDataType); exit(0)

  # Check if this argument allows multiple values to be set.
  # TODO REMOVED WHILE TESTING MULTIPLE INPUT DATA SETS. REMOVE OR REINCLUDE.
#  isMultipleValuesAllowed = graph.CM_getArgumentAttribute(graph.graph, source, target, 'allowMultipleValues')
#  if not isMultipleValuesAllowed and len(values) > 1:
#
#    # Check if this is a top level pipeline argument. Provide the error message dependent on this fact.
#    if longFormArgument in data.longFormArguments.keys():
#      shortFormArgument = data.longFormArguments[longFormArgument].shortFormArgument
#      errors.multipleValues(longFormArgument, shortFormArgument, task = None, isPipeline = True)
#    else:
#      shortFormArgument = graph.CM_getArgumentAttribute(graph.graph, source, target, 'shortFormArgument')
#
#      # Determine the task receiving the argument.
#      if graph.CM_getGraphNodeAttribute(graph.graph, source, 'nodeType') == 'task': task = source
#      else: task = target
#      errors.multipleValues(longFormArgument, shortFormArgument, task = task, isPipeline = False)

  # Loop over each of the values for this node.
  for value in values:

    # Check that the data type is correct.
    #TODO ERROR
    if not isCorrectDataType(value, expectedDataType): print('dataConsistency.checkNode - 2', longFormArgument, value, dataType, type(value)); exit(0)

    # If this is a file, check that the extension is valid. Do not perform this check for stubs.
    if nodeType == 'file':
      if not graph.CM_getArgumentAttribute(graph.graph, source, target, 'isStub'):
        extensions = graph.CM_getArgumentAttribute(graph.graph, source, target, 'extensions')

        # Not all files have specified extensions. If no extensions are supplied, this check should not be performed.
        if extensions:
          task       = target if isInput else source
          fileNodeID = source if isInput else target

          # Fail if there was an error.
          if not checkExtensions(value, extensions):

            # Check if a top level pipeline argument exists.
            if longFormArgument in data.longFormArguments.keys():
              shortFormArgument = data.longFormArguments[longFormArgument].shortFormArgument
              errors.invalidExtensionPipeline(longFormArgument, shortFormArgument, value, extensions)

            # If no pipeline argument exists for this argument, list the task and argument.
            else:
              shortFormArgument = graph.CM_getArgumentAttribute(graph.graph, source, target, 'shortFormArgument')
              errors.invalidExtension(task, longFormArgument, shortFormArgument, value, extensions)

  # Return the expected data type
  return expectedDataType

# Check a values data type.
def isCorrectDataType(value, dataType):

  # If the expected data type is an integer.
  if dataType == 'integer':
    try: value = int(value)
    except: return False
    return True

  # If the expected data type is a float.
  elif dataType == 'float':
    try: value = float(value)
    except: return False
    return True

  # If the expected data type is a string.
  elif dataType == 'string':
    try: value = str(value)
    except: return False
    return True

  # If the expected data type is a Boolean.
  elif dataType == 'bool':
    if value == 'true' or value == 'True' or value == 'false' or value == 'False': return True
    return False

  # If the expected data type is a flag.
  elif dataType == 'flag':
    if value == 'set' or value == 'unset': return True
    return False

  # If none of the above conditions were met, the data type is incorrect.
  return False

# Check that the file extension is valid.
def checkExtensions(value, extensions):

  # If the extensions are not known (e.g. are listed as 'no extension') do not perform this check.
  if len(extensions) == 1 and extensions[0] == 'no extension': return True

  # If the value ends with any of the extensions, return True.
  for extension in extensions:
    if value.endswith(extension): return True

  # Otherwise, return False,
  return False

# Loop over all tasks in the pipeline and check that all required values (excepting output files
# which can be constructed) have been defined.
def checkRequiredArguments(graph, superpipeline, args, isFullCheck):

  # Define error handling,
  errors = er.consistencyErrors()

  # Keep track of nodes that can have their values constructed.
  constructableNodes = []

  # Loop over all tasks in the workflow
  for task in graph.workflow:

    # Get the tool for the task.
    tool     = superpipeline.tasks[task]
    toolData = superpipeline.getToolData(tool)

    # Loop over all of the arguments for the tool and check that all required arguments have a node<commit_msg>Version branch.version2 0.63 - February 2015<commit_after>#!/bin/bash/python

from __future__ import print_function
import dataConsistencyErrors as er
import superpipeline

import json
import os
import sys

# Loop over all of the nodes in a graph and check that the values associated with it
# are of the correct type and have extensions consistent with all of the arguments
# attached to the node.
def checkValues(graph, superpipeline):

  # First, loop over all of the option nodes and check that the data types for all values
  # are valid.
  for nodeID in graph.getNodes(['option', 'file']):
    values   = graph.getGraphNodeAttribute(nodeID, 'values')
    nodeType = graph.getGraphNodeAttribute(nodeID, 'nodeType')

    # Get all of the arguments that use this node and check the data types. Start with all predecessors to this node.
    expectedDataType = None
    for predecessorNodeID in graph.getPredecessors(nodeID):
      expectedDataType = checkNode(graph, superpipeline, predecessorNodeID, nodeID, nodeType, expectedDataType, values, isInput = False)
    for successorNodeID in graph.getSuccessors(nodeID):
      expectedDataType = checkNode(graph, superpipeline, nodeID, successorNodeID, nodeType, expectedDataType, values, isInput = True)

# Check the values for a node.
def checkNode(graph, superpipeline, source, target, nodeType, expectedDataType, values, isInput):
 
  # Define error handling,
  errors = er.consistencyErrors()

  # Get pipeline configuration data.
  data = superpipeline.pipelineConfigurationData[superpipeline.pipeline]

  # Get the required attributes.
  longFormArgument = graph.CM_getArgumentAttribute(graph.graph, source, target, 'longFormArgument')
  dataType         = graph.CM_getArgumentAttribute(graph.graph, source, target, 'dataType')

  # If this is the first argument parsed, populate the expectedDataType variable with the data type for this argument.
  if not expectedDataType: expectedDataType = dataType

  # If expectedDataType is populated and this data type is different to the expectedDataType, this implies that different
  # arguments using the same values expect different data types. This is clearly impossible, so terminate.
  #TODO ERROR
  elif expectedDataType != dataType: print('dataConsistency.checkNode - 1', dataType, expectedDataType); exit(0)

  # Check if this argument allows multiple values to be set.
  # TODO REMOVED WHILE TESTING MULTIPLE INPUT DATA SETS. REMOVE OR REINCLUDE.
#  isMultipleValuesAllowed = graph.CM_getArgumentAttribute(graph.graph, source, target, 'allowMultipleValues')
#  if not isMultipleValuesAllowed and len(values) > 1:
#
#    # Check if this is a top level pipeline argument. Provide the error message dependent on this fact.
#    if longFormArgument in data.longFormArguments.keys():
#      shortFormArgument = data.longFormArguments[longFormArgument].shortFormArgument
#      errors.multipleValues(longFormArgument, shortFormArgument, task = None, isPipeline = True)
#    else:
#      shortFormArgument = graph.CM_getArgumentAttribute(graph.graph, source, target, 'shortFormArgument')
#
#      # Determine the task receiving the argument.
#      if graph.CM_getGraphNodeAttribute(graph.graph, source, 'nodeType') == 'task': task = source
#      else: task = target
#      errors.multipleValues(longFormArgument, shortFormArgument, task = task, isPipeline = False)

  # Loop over each of the values for this node.
  for value in values:

    # Check that the data type is correct.
    #TODO ERROR
    if not isCorrectDataType(value, expectedDataType): print('dataConsistency.checkNode - 2', longFormArgument, value, dataType, type(value)); exit(0)

    # If this is a file, check that the extension is valid. Do not perform this check for stubs.
    if nodeType == 'file':
      if not graph.CM_getArgumentAttribute(graph.graph, source, target, 'isStub'):
        extensions = graph.CM_getArgumentAttribute(graph.graph, source, target, 'extensions')

        # Not all files have specified extensions. If no extensions are supplied, this check should not be performed.
        if extensions:
          task       = target if isInput else source
          fileNodeID = source if isInput else target

          # Fail if there was an error.
          if not checkExtensions(value, extensions):

            # Check if a top level pipeline argument exists.
            if longFormArgument in data.longFormArguments.keys():
              shortFormArgument = data.longFormArguments[longFormArgument].shortFormArgument
              errors.invalidExtensionPipeline(longFormArgument, shortFormArgument, value, extensions)

            # If no pipeline argument exists for this argument, list the task and argument.
            else:
              shortFormArgument = graph.CM_getArgumentAttribute(graph.graph, source, target, 'shortFormArgument')
              errors.invalidExtension(task, longFormArgument, shortFormArgument, value, extensions)

  # Return the expected data type
  return expectedDataType

# Check a values data type.
def isCorrectDataType(value, dataType):

  # If the expected data type is an integer.
  if dataType == 'integer':
    try: value = int(value)
    except: return False
    return True

  # If the expected data type is a float.
  elif dataType == 'float':
    try: value = float(value)
    except: return False
    return True

  # If the expected data type is a string.
  elif dataType == 'string':
    try: value = str(value)
    except: return False
    return True

  # If the expected data type is a Boolean.
  elif dataType == 'bool':
    if value == 'true' or value == 'True' or value == 'false' or value == 'False': return True
    return False

  # If the expected data type is a flag.
  elif dataType == 'flag':
    if value == 'set' or value == 'unset': return True
    return False

  # If none of the above conditions were met, the data type is incorrect.
  return False

# Check that the file extension is valid.
def checkExtensions(value, extensions):

  # If the extensions are not known (e.g. are listed as 'no extension') do not perform this check.
  if len(extensions) == 1 and extensions[0] == 'no extension': return True

  # If the value ends with any of the extensions, return True.
  for extension in extensions:
    if value.endswith(extension): return True

  # Otherwise, return False,
  return False

# Loop over all tasks in the pipeline and check that all required values (excepting output files
# which can be constructed) have been defined.
def checkRequiredArguments(graph, superpipeline, args, isFullCheck):

  # Define error handling,
  errors = er.consistencyErrors()

  # Keep track of nodes that can have their values constructed.
  constructableNodes = []

  # Loop over the defined pipeline arguments and check that all arguments listed as required have
  # been set.
  for argument in args.arguments:
    if args.arguments[argument].isRequired:

      # Loop over the associated graph nodes and see if the values are set.
      for nodeID in args.arguments[argument].graphNodeIDs:

        # If the values haven't been set, terminate. This is a pipeline argument listed as required
        # and so must be set by the user (and not constructed).
        if not graph.getGraphNodeAttribute(nodeID, 'values'):
          shortFormArgument = args.arguments[argument].shortFormArgument
          description       = args.arguments[argument].description
          errors.unsetRequiredArgument(argument, shortFormArgument, description)

  # Loop over all tasks in the workflow
  for task in graph.workflow:

    # Get the tool for the task.
    tool     = superpipeline.tasks[task]
    toolData = superpipeline.getToolData(tool)

    # Loop over all of the arguments for the tool and check that all required arguments have a node
{'fraction_non_alphanumeric': 0.07196969696969698, 'fraction_numerical': 0.001488095238095238, 'mean_word_length': 3.777059773828756, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3844849', 'n_tokens_mistral': 3858, 'n_tokens_neox': 3792, 'n_words': 1592}