<commit_before>package main

import (
	"fmt"
	"path/filepath"
	"time"

	"github.com/compose/transporter/pkg/adaptor"
	"github.com/compose/transporter/pkg/events"
	"github.com/compose/transporter/pkg/transporter"
	"github.com/nu7hatch/gouuid"
	"github.com/robertkrimen/otto"
)

// JavascriptBuilder runs the javascript provided and uses it to compile a
// list of transporter nodes and instantiate a transporter pipeline
type JavascriptBuilder struct {
	file   string
	path   string
	script *otto.Script
	vm     *otto.Otto

	nodes     map[string]Node
	pipelines []*transporter.Pipeline

	err    error
	config Config
}

// NewJavascriptBuilder compiles the supplied javascript and creates a Javascriptbulder
func NewJavascriptBuilder(config Config, file, src string) (*JavascriptBuilder, error) {
	js := &JavascriptBuilder{
		file:      file,
		vm:        otto.New(),
		path:      filepath.Dir(file),
		config:    config,
		nodes:     make(map[string]Node),
		pipelines: make([]*transporter.Pipeline, 0),
	}

	var (
		script *otto.Script
		err    error
	)
	if src != "" {
		script, err = js.vm.Compile("", src)
	} else {
		script, err = js.vm.Compile(file, nil)
	}

	if err != nil {
		return js, err
	}
	js.script = script
	js.vm.Set("Source", js.source)

	return js, nil
}

// source initialize a transporter Node as a source and adds it to the builder's node map.
// Source(..) takes one argument, a javascript hash which generally contains at
// least a name and a namespace property
//   {name: .., namespace: ..}
func (js *JavascriptBuilder) source(call otto.FunctionCall) otto.Value {
	if len(call.ArgumentList) != 1 {
		js.err = fmt.Errorf("Source must be called with 1 arg. (%d given)", len(call.ArgumentList))
		return otto.NullValue()
	}

	node, err := js.findNode(call.Argument(0))
	if err != nil {
		js.err = err
		return otto.NullValue()
	}
	js.nodes[node.UUID] = node // persist this

	nodeObject, err := node.Object()
	if err != nil {
		js.err = err
		return otto.NullValue()
	}

	js.setFunc(nodeObject, "transform", js.transform)
	js.setFunc(nodeObject, "save", js.save)
	return nodeObject.Value()
}

// save adds a sink to the transporter pipeline
// each pipeline can have multiple sinks
func (js *JavascriptBuilder) save(node Node, call otto.FunctionCall) (Node, error) {
	thisNode, err := js.findNode(call.Argument(0))
	if err != nil {
		return node, err
	}
	root := js.nodes[node.RootUUID]

	if node.UUID == root.UUID { // save is being called on a root node
		root.Add(&thisNode)
	} else {
		node.Add(&thisNode) // add the generated not to the `this`
		root.Add(&node)     // add the result to the root
	}

	js.nodes[root.UUID] = root
	return root, err
}

// adds a transform function to the transporter pipeline
// transform takes one argument, which is a path to a transformer file.
func (js *JavascriptBuilder) transform(node Node, call otto.FunctionCall) (Node, error) {
	e, err := call.Argument(0).Export()
	if err != nil {
		return node, err
	}

	rawMap, ok := e.(map[string]interface{})
	if !ok {
		return node, fmt.Errorf("first argument must be an hash. (got %T instead)", e)
	}

	filename, ok := rawMap["filename"].(string)
	if !ok {
		return node, fmt.Errorf("transformer config must contain a valid filename key")
	}
	if !filepath.IsAbs(filename) {
		filename = filepath.Join(js.path, filename)
	}

	debug, ok := rawMap["debug"].(bool)

	name, ok := rawMap["name"].(string)
	if !(ok) {
		u, err := uuid.NewV4()
		if err != nil {
			return node, err
		}
		name = u.String()
	}

	transformer, err := NewNode(name, "transformer", adaptor.Config{"filename": filename, "debug": debug})
	if err != nil {
		return node, err
	}

	node.Add(&transformer)

	return transformer, nil
}

// pipelines in javascript are chainable, you take in a pipeline, and you return a pipeline
// we just generalize some of that logic here
func (js *JavascriptBuilder) setFunc(obj *otto.Object, token string, fn func(Node, otto.FunctionCall) (Node, error)) error {
	return obj.Set(token, func(call otto.FunctionCall) otto.Value {
		this, _ := call.This.Export()

		node, err := CreateNode(this)
		if err != nil {
			js.err = err
			return otto.NullValue()
		}

		node, err = fn(node, call)
		if err != nil {<commit_msg>add some more information in errors that we get in the javascript. fixes #24<commit_after>package main

import (
	"fmt"
	"path/filepath"
	"time"

	"github.com/compose/transporter/pkg/adaptor"
	"github.com/compose/transporter/pkg/events"
	"github.com/compose/transporter/pkg/transporter"
	"github.com/nu7hatch/gouuid"
	"github.com/robertkrimen/otto"
)

// JavascriptBuilder runs the javascript provided and uses it to compile a
// list of transporter nodes and instantiate a transporter pipeline
type JavascriptBuilder struct {
	file   string
	path   string
	script *otto.Script
	vm     *otto.Otto

	nodes     map[string]Node
	pipelines []*transporter.Pipeline

	err    error
	config Config
}

// NewJavascriptBuilder compiles the supplied javascript and creates a Javascriptbulder
func NewJavascriptBuilder(config Config, file, src string) (*JavascriptBuilder, error) {
	js := &JavascriptBuilder{
		file:      file,
		vm:        otto.New(),
		path:      filepath.Dir(file),
		config:    config,
		nodes:     make(map[string]Node),
		pipelines: make([]*transporter.Pipeline, 0),
	}

	var (
		script *otto.Script
		err    error
	)
	if src != "" {
		script, err = js.vm.Compile("", src)
	} else {
		script, err = js.vm.Compile(file, nil)
	}

	if err != nil {
		return js, err
	}
	js.script = script
	js.vm.Set("Source", js.source)

	return js, nil
}

// source initialize a transporter Node as a source and adds it to the builder's node map.
// Source(..) takes one argument, a javascript hash which generally contains at
// least a name and a namespace property
//   {name: .., namespace: ..}
func (js *JavascriptBuilder) source(call otto.FunctionCall) otto.Value {
	if len(call.ArgumentList) != 1 {
		js.err = fmt.Errorf("Source must be called with 1 arg. (%d given)", len(call.ArgumentList))
		return otto.NullValue()
	}

	node, err := js.findNode(call.Argument(0))
	if err != nil {
		js.err = fmt.Errorf("source error, %s", err.Error())
		return otto.NullValue()
	}
	js.nodes[node.UUID] = node // persist this

	nodeObject, err := node.Object()
	if err != nil {
		js.err = err
		return otto.NullValue()
	}

	js.setFunc(nodeObject, "transform", js.transform)
	js.setFunc(nodeObject, "save", js.save)
	return nodeObject.Value()
}

// save adds a sink to the transporter pipeline
// each pipeline can have multiple sinks
func (js *JavascriptBuilder) save(node Node, call otto.FunctionCall) (Node, error) {
	thisNode, err := js.findNode(call.Argument(0))
	if err != nil {
		return node, fmt.Errorf("save error, %s", err.Error())
	}
	root := js.nodes[node.RootUUID]

	if node.UUID == root.UUID { // save is being called on a root node
		root.Add(&thisNode)
	} else {
		node.Add(&thisNode) // add the generated not to the `this`
		root.Add(&node)     // add the result to the root
	}

	js.nodes[root.UUID] = root
	return root, nil
}

// adds a transform function to the transporter pipeline
// transform takes one argument, which is a path to a transformer file.
func (js *JavascriptBuilder) transform(node Node, call otto.FunctionCall) (Node, error) {
	e, err := call.Argument(0).Export()
	if err != nil {
		return node, err
	}

	rawMap, ok := e.(map[string]interface{})
	if !ok {
		return node, fmt.Errorf("transform error. first argument must be an hash. (got %T instead)", e)
	}

	filename, ok := rawMap["filename"].(string)
	if !ok {
		return node, fmt.Errorf("transformer config must contain a valid filename key")
	}
	if !filepath.IsAbs(filename) {
		filename = filepath.Join(js.path, filename)
	}

	debug, ok := rawMap["debug"].(bool)

	name, ok := rawMap["name"].(string)
	if !(ok) {
		u, err := uuid.NewV4()
		if err != nil {
			return node, fmt.Errorf("transform error. uuid error (%s)", err.Error())
		}
		name = u.String()
	}

	transformer, err := NewNode(name, "transformer", adaptor.Config{"filename": filename, "debug": debug})
	if err != nil {
		return node, fmt.Errorf("transform error. cannot create node (%s)", err.Error())
	}

	node.Add(&transformer)

	return transformer, nil
}

// pipelines in javascript are chainable, you take in a pipeline, and you return a pipeline
// we just generalize some of that logic here
func (js *JavascriptBuilder) setFunc(obj *otto.Object, token string, fn func(Node, otto.FunctionCall) (Node, error)) error {
	return obj.Set(token, func(call otto.FunctionCall) otto.Value {
		this, _ := call.This.Export()

		node, err := CreateNode(this)
		if err != nil {
			js.err = err
			return otto.NullValue()
		}

		node, err = fn(node, call)
		if err != nil {
{'fraction_non_alphanumeric': 0.13239339752407153, 'fraction_numerical': 0.0020632737276478678, 'mean_word_length': 3.6533333333333333, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5547627', 'n_tokens_mistral': 3058, 'n_tokens_neox': 2882, 'n_words': 913}