<commit_before>package cwl

import (
	"fmt"
	"io/ioutil"
	"path"
	"reflect"
	"strings"

	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/davecgh/go-spew/spew"
)

// GraphDocument this is used by YAML or JSON library for inital parsing
type GraphDocument struct {
	CwlVersion CWLVersion        `yaml:"cwlVersion,omitempty" json:"cwlVersion,omitempty" bson:"cwlVersion,omitempty" mapstructure:"cwlVersion,omitempty"`
	Base       interface{}       `yaml:"$base,omitempty" json:"$base,omitempty" bson:"base,omitempty" mapstructure:"$base,omitempty"`
	Graph      []interface{}     `yaml:"$graph" json:"$graph" bson:"graph" mapstructure:"$graph"` // can only be used for reading, yaml has problems writing interface objetcs
	Namespaces map[string]string `yaml:"$namespaces,omitempty" json:"$namespaces,omitempty" bson:"namespaces,omitempty" mapstructure:"$namespaces,omitempty"`
	Schemas    []interface{}     `yaml:"$schemas,omitempty" json:"$schemas,omitempty" bson:"schemas,omitempty" mapstructure:"$schemas,omitempty"`
}

// ParseCWLGraphDocument _
// entrypoint defauyts to #main but can anything else...
func ParseCWLGraphDocument(yamlStr string, entrypoint string, context *WorkflowContext) (objectArray []NamedCWLObject, schemata []CWLType_Type, schemas []interface{}, err error) {

	cwlGen := GraphDocument{}

	yamlByte := []byte(yamlStr)
	err = Unmarshal(&yamlByte, &cwlGen)
	if err != nil {
		logger.Debug(1, "CWL unmarshal error")
		err = fmt.Errorf("(Parse_cwl_graph_document) Unmarshal returned: " + err.Error())
		return
	}
	//fmt.Println("-------------- yamlStr")
	//fmt.Println(yamlStr)
	//fmt.Println("-------------- raw CWL")
	//spew.Dump(cwl_gen)
	//fmt.Println("-------------- Start parsing")

	if len(cwlGen.Graph) == 0 {
		err = fmt.Errorf("(Parse_cwl_graph_document) len(cwlGen.Graph) == 0")
		return
	}

	// resolve $import
	//var new_obj []interface{}
	//err = Resolve_ImportsArray(cwl_gen.Graph, context)
	//if err != nil {
	//	err = fmt.Errorf("(Parse_cwl_graph_document) Resolve_Imports returned: " + err.Error())
	//	return
	//}

	//fmt.Println("############# cwl_gen.Graph:")
	//spew.Dump(cwl_gen.Graph)
	//panic("done")
	//if new_obj != nil {
	//	err = fmt.Errorf("(Parse_cwl_graph_document) $import in $graph not supported yet")
	//	return
	//}

	//context.Graph = cwlGen.Graph
	context.GraphDocument = cwlGen
	context.CwlVersion = cwlGen.CwlVersion

	if cwlGen.Namespaces != nil {
		context.Namespaces = cwlGen.Namespaces
	}

	if cwlGen.Schemas != nil {
		schemas = cwlGen.Schemas
	}

	// iterate over Graph

	// try to find CWL version!
	if context.CwlVersion == "" {
		for _, elem := range cwlGen.Graph {
			elemMap, ok := elem.(map[string]interface{})
			if ok {
				cwlVersionIf, hasVersion := elemMap["cwlVersion"]
				if hasVersion {

					var cwlVersionStr string
					cwlVersionStr, ok = cwlVersionIf.(string)
					if !ok {
						err = fmt.Errorf("(Parse_cwl_graph_document) Could not read CWLVersion (%s)", reflect.TypeOf(cwlVersionIf))
						return
					}
					context.CwlVersion = CWLVersion(cwlVersionStr)
					break
				}

			}
		}

	}

	if context.CwlVersion == "" {
		// see raw
		err = fmt.Errorf("(Parse_cwl_graph_document) cwl_version empty")
		return
	}

	//fmt.Println("-------------- A Parse_cwl_document")
	if entrypoint == "" {
		entrypoint = "#main"
	}

	if !strings.HasPrefix(entrypoint, "#") {
		err = fmt.Errorf("(Parse_cwl_graph_document) entrypoint not absolute: %s", entrypoint)
		return
	}

	err = context.Init(entrypoint)
	if err != nil {
		err = fmt.Errorf("(Parse_cwl_graph_document) (using entrypoint=%s) context.Init returned: %s", entrypoint, err.Error())
		return
	}

	for id, object := range context.Objects {
		namedObj := NewNamedCWLObject(id, object)
		objectArray = append(objectArray, namedObj)
	}

	// object_array []NamedCWLObject
	//fmt.Println("############# object_array:")
	//spew.Dump(object_array)

	return
}

// ParseCWLSimpleDocument _
// returns: objectID may be used an entrypoint later
func ParseCWLSimpleDocument(yamlStr string, basename string, context *WorkflowContext) (objectArray []NamedCWLObject, schemata []CWLType_Type, schemas []interface{}, objectID string, err error) {
	// Here I expect a single object, Workflow or CommandLIneTool
	//fmt.Printf("-------------- yaml_str: %s\n", yaml_str)

	//fmt.Printf("ParseCWLSimpleDocument start\n")
	//defer fmt.Printf("ParseCWLSimpleDocument end\n")

	var objectIf map[string]interface{}

	yamlByte := []byte(yamlStr)
	err = Unmarshal(&yamlByte, &objectIf)
	if err != nil {
		//logger.Debug(1, "CWL unmarshal error")
		err = fmt.Errorf("(ParseCWLSimpleDocument) Unmarshal returns: %s", err.Error())
		return
	}
	//fmt.Println("objectIf:")
	//spew.Dump(objectIf)
	var objectIfTranslated interface{}
	objectIfTranslated, err = translate(objectIf, context.Path)
	if err != nil {
		err = fmt.Errorf("(ParseCWLSimpleDocument) translate returned: %s", err.Error())
		return
	}
	//fmt.Println("objectIfTranslated:")
	//spew.Dump(objectIfTranslated)
	var ok bool
	objectIf = nil
	objectIf, ok = objectIfTranslated.(map[string]interface{})
	if !ok {
		err = fmt.Errorf("(ParseCWLSimpleDocument) could not translate: %s", err.Error())
		return
	}

	if context.CwlVersion == "" {

		cwlVersionIf, hasVersion := objectIf["cwlVersion"]
		if hasVersion {

			cwlVersionStr, ok := cwlVersionIf.(string)
			if ok {
				context.CwlVersion = NewCWLVersion(cwlVersionStr)
			} else {
				err = fmt.Errorf("(ParseCWLSimpleDocument) version not string (type: %s)", reflect.TypeOf(cwlVersionIf))
				return
			}
		} else {
			spew.Dump(objectIf)
			err = fmt.Errorf("(ParseCWLSimpleDocument) no version found")
			return
		}
	}

	if objectIf == nil {
		err = fmt.Errorf("(ParseCWLSimpleDocument) objectIf == nil")
		return
	}

	//fmt.Println("object_if:")
	//spew.Dump(object_if)
	//var ok bool
	var namespacesIf interface{}
	namespacesIf, ok = objectIf["$namespaces"]
	if ok {
		var namespacesMap map[string]interface{}
		namespacesMap, ok = namespacesIf.(map[string]interface{})
		if !ok {
			err = fmt.Errorf("(ParseCWLSimpleDocument) namespaces_if error (type: %s)", reflect.TypeOf(namespacesIf))<commit_msg>fix namespaces parsing<commit_after>package cwl

import (
	"fmt"
	"io/ioutil"
	"path"
	"reflect"
	"strings"

	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/davecgh/go-spew/spew"
)

// GraphDocument this is used by YAML or JSON library for inital parsing
type GraphDocument struct {
	CwlVersion CWLVersion        `yaml:"cwlVersion,omitempty" json:"cwlVersion,omitempty" bson:"cwlVersion,omitempty" mapstructure:"cwlVersion,omitempty"`
	Base       interface{}       `yaml:"$base,omitempty" json:"$base,omitempty" bson:"base,omitempty" mapstructure:"$base,omitempty"`
	Graph      []interface{}     `yaml:"$graph" json:"$graph" bson:"graph" mapstructure:"$graph"` // can only be used for reading, yaml has problems writing interface objetcs
	Namespaces map[string]string `yaml:"$namespaces,omitempty" json:"$namespaces,omitempty" bson:"namespaces,omitempty" mapstructure:"$namespaces,omitempty"`
	Schemas    []interface{}     `yaml:"$schemas,omitempty" json:"$schemas,omitempty" bson:"schemas,omitempty" mapstructure:"$schemas,omitempty"`
}

// ParseCWLGraphDocument _
// entrypoint defauyts to #main but can anything else...
func ParseCWLGraphDocument(yamlStr string, entrypoint string, context *WorkflowContext) (objectArray []NamedCWLObject, schemata []CWLType_Type, schemas []interface{}, err error) {

	cwlGen := GraphDocument{}

	yamlByte := []byte(yamlStr)
	err = Unmarshal(&yamlByte, &cwlGen)
	if err != nil {
		logger.Debug(1, "CWL unmarshal error")
		err = fmt.Errorf("(Parse_cwl_graph_document) Unmarshal returned: " + err.Error())
		return
	}
	//fmt.Println("-------------- yamlStr")
	//fmt.Println(yamlStr)
	//fmt.Println("-------------- raw CWL")
	//spew.Dump(cwl_gen)
	//fmt.Println("-------------- Start parsing")

	if len(cwlGen.Graph) == 0 {
		err = fmt.Errorf("(Parse_cwl_graph_document) len(cwlGen.Graph) == 0")
		return
	}

	// resolve $import
	//var new_obj []interface{}
	//err = Resolve_ImportsArray(cwl_gen.Graph, context)
	//if err != nil {
	//	err = fmt.Errorf("(Parse_cwl_graph_document) Resolve_Imports returned: " + err.Error())
	//	return
	//}

	//fmt.Println("############# cwl_gen.Graph:")
	//spew.Dump(cwl_gen.Graph)
	//panic("done")
	//if new_obj != nil {
	//	err = fmt.Errorf("(Parse_cwl_graph_document) $import in $graph not supported yet")
	//	return
	//}

	//context.Graph = cwlGen.Graph
	context.GraphDocument = cwlGen
	context.CwlVersion = cwlGen.CwlVersion

	if cwlGen.Namespaces != nil {
		context.Namespaces = cwlGen.Namespaces
	}

	if cwlGen.Schemas != nil {
		schemas = cwlGen.Schemas
	}

	// iterate over Graph

	// try to find CWL version!
	if context.CwlVersion == "" {
		for _, elem := range cwlGen.Graph {
			elemMap, ok := elem.(map[string]interface{})
			if ok {
				cwlVersionIf, hasVersion := elemMap["cwlVersion"]
				if hasVersion {

					var cwlVersionStr string
					cwlVersionStr, ok = cwlVersionIf.(string)
					if !ok {
						err = fmt.Errorf("(Parse_cwl_graph_document) Could not read CWLVersion (%s)", reflect.TypeOf(cwlVersionIf))
						return
					}
					context.CwlVersion = CWLVersion(cwlVersionStr)
					break
				}

			}
		}

	}

	if context.CwlVersion == "" {
		// see raw
		err = fmt.Errorf("(Parse_cwl_graph_document) cwl_version empty")
		return
	}

	//fmt.Println("-------------- A Parse_cwl_document")
	if entrypoint == "" {
		entrypoint = "#main"
	}

	if !strings.HasPrefix(entrypoint, "#") {
		err = fmt.Errorf("(Parse_cwl_graph_document) entrypoint not absolute: %s", entrypoint)
		return
	}

	err = context.Init(entrypoint)
	if err != nil {
		err = fmt.Errorf("(Parse_cwl_graph_document) (using entrypoint=%s) context.Init returned: %s", entrypoint, err.Error())
		return
	}

	for id, object := range context.Objects {
		namedObj := NewNamedCWLObject(id, object)
		objectArray = append(objectArray, namedObj)
	}

	// object_array []NamedCWLObject
	//fmt.Println("############# object_array:")
	//spew.Dump(object_array)

	return
}

// ParseCWLSimpleDocument _
// returns: objectID may be used an entrypoint later
func ParseCWLSimpleDocument(yamlStr string, basename string, context *WorkflowContext) (objectArray []NamedCWLObject, schemata []CWLType_Type, schemas []interface{}, objectID string, err error) {
	// Here I expect a single object, Workflow or CommandLIneTool
	//fmt.Printf("-------------- yaml_str: %s\n", yaml_str)

	//fmt.Printf("ParseCWLSimpleDocument start\n")
	//defer fmt.Printf("ParseCWLSimpleDocument end\n")

	var objectIf map[string]interface{}

	yamlByte := []byte(yamlStr)
	err = Unmarshal(&yamlByte, &objectIf)
	if err != nil {
		//logger.Debug(1, "CWL unmarshal error")
		err = fmt.Errorf("(ParseCWLSimpleDocument) Unmarshal returns: %s", err.Error())
		return
	}
	//fmt.Println("objectIf:")
	//spew.Dump(objectIf)
	var objectIfTranslated interface{}
	objectIfTranslated, err = translate(objectIf, context.Path)
	if err != nil {
		err = fmt.Errorf("(ParseCWLSimpleDocument) translate returned: %s", err.Error())
		return
	}
	//fmt.Println("objectIfTranslated:")
	//spew.Dump(objectIfTranslated)
	var ok bool
	objectIf = nil
	objectIf, ok = objectIfTranslated.(map[string]interface{})
	if !ok {
		err = fmt.Errorf("(ParseCWLSimpleDocument) could not translate: %s", err.Error())
		return
	}

	if context.CwlVersion == "" {

		cwlVersionIf, hasVersion := objectIf["cwlVersion"]
		if hasVersion {

			cwlVersionStr, ok := cwlVersionIf.(string)
			if ok {
				context.CwlVersion = NewCWLVersion(cwlVersionStr)
			} else {
				err = fmt.Errorf("(ParseCWLSimpleDocument) version not string (type: %s)", reflect.TypeOf(cwlVersionIf))
				return
			}
		} else {
			spew.Dump(objectIf)
			err = fmt.Errorf("(ParseCWLSimpleDocument) no version found")
			return
		}
	}

	if objectIf == nil {
		err = fmt.Errorf("(ParseCWLSimpleDocument) objectIf == nil")
		return
	}

	//fmt.Println("object_if:")
	//spew.Dump(object_if)
	//var ok bool
	var namespacesIf interface{}
	namespacesIf, ok = objectIf["$namespaces"]
	if ok {

		namespacesIf, err = MakeStringMap(namespacesIf, nil)
		if err != nil {
			return
		}

		var namespacesMap map[string]interface{}
		namespacesMap, ok = namespacesIf.(map[string]interface{})
		if !ok {
			err = fmt.Errorf("(ParseCWLSimpleDocument) namespaces_if error (type: %s)", reflect.TypeOf(namespacesIf))
{'fraction_non_alphanumeric': 0.1556580864497329, 'fraction_numerical': 0.0006475635421725757, 'mean_word_length': 4.143630308076603, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7292491', 'n_tokens_mistral': 4573, 'n_tokens_neox': 4190, 'n_words': 915}