<commit_before>// Copyright 2017-2018, Square, Inc.

package grapher

import (
	"fmt"
	"reflect"
	"strings"

	"github.com/square/spincycle/job"
	"github.com/square/spincycle/request-manager/id"
)

// The Grapher struct contains the sequence specs required to construct graphs.
// The user must handle the creation of the Sequence Specs.
//
// CreateGraph will create a graph. The user must provide a Sequence Type, to indicate
// what graph will be created.
type Grapher struct {
	AllSequences map[string]*SequenceSpec // All sequences that were read in from the Config
	JobFactory   job.Factory              // factory to create nodes' jobs.

	idgen id.Generator // Generates UIDs for the nodes created by the Grapher.
}

// NewGrapher returns a new Grapher struct. The caller of NewGrapher mut provide
// a Job Factory for Grapher to create the jobs that will be stored at each node.
// An id generator must also be provided (used for generating ids for nodes).
//
// A new Grapher should be made for every request.
func NewGrapher(nf job.Factory, cfg *Config, idgen id.Generator) *Grapher {
	o := &Grapher{
		JobFactory:   nf,
		AllSequences: cfg.Sequences,
		idgen:        idgen,
	}
	return o
}

// A GrapherFactory makes Graphers.
type GrapherFactory interface {
	// Make makes a Grapher. A new grapher should be made for every request.
	Make() *Grapher
}

// grapherFactory implements the GrapherFactory interface.
type grapherFactory struct {
	jf     job.Factory
	config *Config
	idf    id.GeneratorFactory
}

// NewGrapherFactory creates a GrapherFactory.
func NewGrapherFactory(jf job.Factory, cfg *Config, idf id.GeneratorFactory) GrapherFactory {
	return &grapherFactory{
		jf:     jf,
		config: cfg,
		idf:    idf,
	}
}

func (gf *grapherFactory) Make() *Grapher {
	return NewGrapher(gf.jf, gf.config, gf.idf.Make()) // create a Grapher with a new id Generator
}

// CreateGraph will create a graph. The user must provide a Sequence Name, to indicate
// what graph will be created. The caller must also provide the first set of args.
func (o *Grapher) CreateGraph(sequenceName string, args map[string]interface{}) (*Graph, error) {
	request, ok := o.AllSequences[sequenceName]
	if !ok {
		return nil, fmt.Errorf("cannot find definition for request: %s", sequenceName)
	}
	g, err := o.buildSequence(request.Name, request, args)
	if err != nil {
		return nil, err
	}

	return g, nil
}

func (g *Grapher) Sequences() map[string]*SequenceSpec {
	return g.AllSequences
}

// buildSequence will take in a sequence spec and return a Graph that represents the sequence
func (o *Grapher) buildSequence(name string, seq *SequenceSpec, args map[string]interface{}) (*Graph, error) {

	// Verify all required arguments are present
	for _, arg := range seq.Args.Required {
		if _, ok := args[arg.Name]; !ok {
			return nil, fmt.Errorf("sequence %s missing arg %s", name, arg.Name)
		}
	}

	// Verify all optional arguments with defaults provided are included
	for _, arg := range seq.Args.Optional {
		if _, ok := args[arg.Name]; !ok {
			args[arg.Name] = arg.Default
		}
	}

	// Verify all static arguments with defaults provided are included
	for _, arg := range seq.Args.Static {
		if _, ok := args[arg.Name]; !ok {
			args[arg.Name] = arg.Default
		}
	}
	return o.buildComponent("sequence_"+name, seq.Nodes, args, seq.Retry)
}

// buildComponent, given a set of node specs, and node args, create a graph that represents the list of node specs,
// nodeArgs represents the arguments that will be passed into the nodes on creation
func (o *Grapher) buildComponent(name string, nodeDefs map[string]*NodeSpec, nodeArgs map[string]interface{}, sequenceRetry uint) (*Graph, error) {

	// Start with an empty graph. Enforce a
	// single source and a single sink node here.
	g, err := o.newEmptyGraph(name, nodeArgs)
	if err != nil {
		return nil, err
	}

	// Store configured retry from sequence spec on the first node in the
	// sequence
	g.First.SequenceRetry = sequenceRetry

	//////////////////////////////////////////////////////
	// First, construct all subcomponents of this graph

	// components is a map of all sub-components of this graph
	components := map[*NodeSpec]*Graph{}

	// nodesToBeDone is a map of all jobs that have not yet been built
	nodesToBeDone := map[*NodeSpec]bool{}
	for _, node := range nodeDefs {
		nodesToBeDone[node] = true
	}
	nodesCount := len(nodesToBeDone)

	for nodesCount > 0 {

		for n, _ := range nodesToBeDone {

			// First check that all arguments required for the job are present.
			// If not all arguments are present, then find another node to construct
			if !o.allArgsPresent(n, nodeArgs) {
				continue
			}

			// Find out how many times this node has to be repeated
			iterators, iterateOvers, err := o.getIterators(n, nodeArgs)
			if err != nil {
				return nil, err
			}

			// All the graphs that make up this component
			componentsForThisNode := []*Graph{}

			// Validate that the iterators all have variable names
			if len(iterateOvers) != len(iterators) || len(iterators) < 1 {
				return nil, fmt.Errorf("Error making node %s: malformed arguments given.", n.Name)
			}

			// If no repetition is needed, this loop will only execute once
			for i, _ := range iterateOvers[0] {

				// Copy the required args into a separate args map here.
				// Do the necessary remapping here.
				nodeArgsCopy, err := o.remapNodeArgs(n, nodeArgs)
				if err != nil {
					return nil, err
				}

				// Add the iterator to the node args unless there is no iterator for this node
				for j, iterator := range iterators {
					if iterator != "" {
						// This won't panic because we have earlier asserted that
						// len(iterators) == len(iterateOvers)
						nodeArgsCopy[iterator] = iterateOvers[j][i]
					}
				}

				// Build next graph component and assert that it's valid
				var g *Graph
				if !n.isSequence() {
					// Node is a job, create a graph that contains only the node
					g, err = o.buildSingleVertexGraph(n, nodeArgsCopy)
					if err != nil {
						return nil, err
					}
				} else {
					// Node is a sequence, recursively construct its components
					sequence, ok := o.AllSequences[n.NodeType]
					if !ok {
						return nil, fmt.Errorf("could not find sequence %s", name)
					}
					g, err = o.buildSequence(n.Name, sequence, nodeArgsCopy)
					if err != nil {
						return nil, err
					}
				}
				if !g.IsValidGraph() {
					return nil, fmt.Errorf("malformed graph created")
				}

				// Add the new job to the map of completed components
				componentsForThisNode = append(componentsForThisNode, g)

				// If the node (or sequence) was determined to set any args
				// copy them from nodeArgsCopy into the main nodeArgs
				err = o.setNodeArgs(n, nodeArgs, nodeArgsCopy)
				if err != nil {
					return nil, err<commit_msg>sequence retry set by request<commit_after>// Copyright 2017-2018, Square, Inc.

package grapher

import (
	"fmt"
	"reflect"
	"strings"

	"github.com/square/spincycle/job"
	"github.com/square/spincycle/request-manager/id"
)

// The Grapher struct contains the sequence specs required to construct graphs.
// The user must handle the creation of the Sequence Specs.
//
// CreateGraph will create a graph. The user must provide a Sequence Type, to indicate
// what graph will be created.
type Grapher struct {
	AllSequences map[string]*SequenceSpec // All sequences that were read in from the Config
	JobFactory   job.Factory              // factory to create nodes' jobs.

	idgen id.Generator // Generates UIDs for the nodes created by the Grapher.
}

// NewGrapher returns a new Grapher struct. The caller of NewGrapher mut provide
// a Job Factory for Grapher to create the jobs that will be stored at each node.
// An id generator must also be provided (used for generating ids for nodes).
//
// A new Grapher should be made for every request.
func NewGrapher(nf job.Factory, cfg *Config, idgen id.Generator) *Grapher {
	o := &Grapher{
		JobFactory:   nf,
		AllSequences: cfg.Sequences,
		idgen:        idgen,
	}
	return o
}

// A GrapherFactory makes Graphers.
type GrapherFactory interface {
	// Make makes a Grapher. A new grapher should be made for every request.
	Make() *Grapher
}

// grapherFactory implements the GrapherFactory interface.
type grapherFactory struct {
	jf     job.Factory
	config *Config
	idf    id.GeneratorFactory
}

// NewGrapherFactory creates a GrapherFactory.
func NewGrapherFactory(jf job.Factory, cfg *Config, idf id.GeneratorFactory) GrapherFactory {
	return &grapherFactory{
		jf:     jf,
		config: cfg,
		idf:    idf,
	}
}

func (gf *grapherFactory) Make() *Grapher {
	return NewGrapher(gf.jf, gf.config, gf.idf.Make()) // create a Grapher with a new id Generator
}

// CreateGraph will create a graph. The user must provide a Sequence Name, to indicate
// what graph will be created. The caller must also provide the first set of args.
func (o *Grapher) CreateGraph(sequenceName string, args map[string]interface{}) (*Graph, error) {
	request, ok := o.AllSequences[sequenceName]
	if !ok {
		return nil, fmt.Errorf("cannot find definition for request: %s", sequenceName)
	}
	g, err := o.buildSequence(request.Name, request, args)
	if err != nil {
		return nil, err
	}

	return g, nil
}

func (g *Grapher) Sequences() map[string]*SequenceSpec {
	return g.AllSequences
}

// buildSequence will take in a sequence spec and return a Graph that represents the sequence
func (o *Grapher) buildSequence(name string, seq *SequenceSpec, args map[string]interface{}) (*Graph, error) {

	// Verify all required arguments are present
	for _, arg := range seq.Args.Required {
		if _, ok := args[arg.Name]; !ok {
			return nil, fmt.Errorf("sequence %s missing arg %s", name, arg.Name)
		}
	}

	// Verify all optional arguments with defaults provided are included
	for _, arg := range seq.Args.Optional {
		if _, ok := args[arg.Name]; !ok {
			args[arg.Name] = arg.Default
		}
	}

	// Verify all static arguments with defaults provided are included
	for _, arg := range seq.Args.Static {
		if _, ok := args[arg.Name]; !ok {
			args[arg.Name] = arg.Default
		}
	}
	return o.buildComponent("sequence_"+name, seq.Nodes, args, seq.Retry)
}

// buildComponent, given a set of node specs, and node args, create a graph that represents the list of node specs,
// nodeArgs represents the arguments that will be passed into the nodes on creation
func (o *Grapher) buildComponent(name string, nodeDefs map[string]*NodeSpec, nodeArgs map[string]interface{}, sequenceRetry uint) (*Graph, error) {

	// Start with an empty graph. Enforce a
	// single source and a single sink node here.
	g, err := o.newEmptyGraph(name, nodeArgs)
	if err != nil {
		return nil, err
	}

	// Store configured retry from sequence spec on the first node in the
	// sequence
	g.First.SequenceRetry = sequenceRetry

	//////////////////////////////////////////////////////
	// First, construct all subcomponents of this graph

	// components is a map of all sub-components of this graph
	components := map[*NodeSpec]*Graph{}

	// nodesToBeDone is a map of all jobs that have not yet been built
	nodesToBeDone := map[*NodeSpec]bool{}
	for _, node := range nodeDefs {
		nodesToBeDone[node] = true
	}
	nodesCount := len(nodesToBeDone)

	for nodesCount > 0 {

		for n, _ := range nodesToBeDone {

			// First check that all arguments required for the job are present.
			// If not all arguments are present, then find another node to construct
			if !o.allArgsPresent(n, nodeArgs) {
				continue
			}

			// Find out how many times this node has to be repeated
			iterators, iterateOvers, err := o.getIterators(n, nodeArgs)
			if err != nil {
				return nil, err
			}

			// All the graphs that make up this component
			componentsForThisNode := []*Graph{}

			// Validate that the iterators all have variable names
			if len(iterateOvers) != len(iterators) || len(iterators) < 1 {
				return nil, fmt.Errorf("Error making node %s: malformed arguments given.", n.Name)
			}

			// If no repetition is needed, this loop will only execute once
			for i, _ := range iterateOvers[0] {

				// Copy the required args into a separate args map here.
				// Do the necessary remapping here.
				nodeArgsCopy, err := o.remapNodeArgs(n, nodeArgs)
				if err != nil {
					return nil, err
				}

				// Add the iterator to the node args unless there is no iterator for this node
				for j, iterator := range iterators {
					if iterator != "" {
						// This won't panic because we have earlier asserted that
						// len(iterators) == len(iterateOvers)
						nodeArgsCopy[iterator] = iterateOvers[j][i]
					}
				}

				// Build next graph component and assert that it's valid
				var g *Graph
				if !n.isSequence() {
					// Node is a job, create a graph that contains only the node
					g, err = o.buildSingleVertexGraph(n, nodeArgsCopy)
					if err != nil {
						return nil, err
					}
				} else {
					// Node is a sequence, recursively construct its components
					sequence, ok := o.AllSequences[n.NodeType]
					if !ok {
						return nil, fmt.Errorf("could not find sequence %s", name)
					}
					sequence.Retry = n.Retry
					g, err = o.buildSequence(n.Name, sequence, nodeArgsCopy)
					if err != nil {
						return nil, err
					}
				}
				if !g.IsValidGraph() {
					return nil, fmt.Errorf("malformed graph created")
				}

				// Add the new job to the map of completed components
				componentsForThisNode = append(componentsForThisNode, g)

				// If the node (or sequence) was determined to set any args
				// copy them from nodeArgsCopy into the main nodeArgs
				err = o.setNodeArgs(n, nodeArgs, nodeArgsCopy)
				if err != nil {
					return nil, err
{'fraction_non_alphanumeric': 0.10297566695983583, 'fraction_numerical': 0.0016124303723248313, 'mean_word_length': 3.1161387631975868, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '650736', 'n_tokens_mistral': 4590, 'n_tokens_neox': 4057, 'n_words': 1607}