<commit_before>package main

import (
	"bytes"
	"fmt"

	r "reflect"

	"strings"

	sci "github.com/samuell/scipipe"
)

func main() {
	p := NewFooToBarReplacer()
	fmt.Println("Process: ", p)
}

type FooToBarReplacer struct {
	sci.Process
	InnerProcess *sci.Process
	InFoo        chan *sci.FileTarget
	OutBar       chan *sci.FileTarget
}

func (p *FooToBarReplacer) Run() {
	// ...
}

func NewFooToBarReplacer() interface{} {
	execFunc := func(task *sci.ShellTask) {
		indata := task.InTargets["foo"].Read()
		indataReplaced := bytes.Replace(indata, []byte("foo"), []byte("bar"), -1)
		task.OutTargets["bar"].WriteTempFile(indataReplaced)
	}
	pathFuncs := map[string]func(*sci.ShellTask) string{
		"bar": func(t *sci.ShellTask) string { return t.InTargets["foo"].GetPath() + ".bar.txt" },
	}
	return NewProcessFromStruct(&FooToBarReplacer{}, execFunc, pathFuncs)
}

func NewProcessFromStruct(procStruct interface{}, execFunc func(*sci.ShellTask), pathFuncs map[string]func(*sci.ShellTask) string) interface{} {
	// Get in-ports of struct
	procStructVal := r.ValueOf(procStruct).Elem()
	for i := 0; i < procStructVal.NumField(); i++ {
		structFieldName := procStructVal.Type().Field(i).Name
		structFieldType := procStructVal.Type().Field(i).Type
		exampleChan := make(chan *sci.FileTarget)
		if strings.HasPrefix(structFieldName, "In") && structFieldType == r.TypeOf(exampleChan) {
			fmt.Println("In-port:", structFieldName)
		} else if strings.HasPrefix(structFieldName, "Out") && structFieldType == r.TypeOf(exampleChan) {
			fmt.Println("Out-port:", structFieldName)
		}
	}
	return procStruct
}
<commit_msg>Collecting in- and out-ports, in example 16<commit_after>package main

import (
	"bytes"
	"fmt"

	r "reflect"

	"strings"

	sci "github.com/samuell/scipipe"
)

func main() {
	p := NewFooToBarReplacer()
	fmt.Println("Process: ", p)
}

// -------------------------------------------
//  Example of defining a new wrapper task
// -------------------------------------------

type FooToBarReplacer struct {
	sci.Process
	InnerProcess *sci.Process
	Run          func(p *FooToBarReplacer)
	InFoo        chan *sci.FileTarget
	OutBar       chan *sci.FileTarget
}

func (p *FooToBarReplacer) Run() {
	// ...
}

func NewFooToBarReplacer() interface{} {
	execFunc := func(task *sci.ShellTask) {
		indata := task.InTargets["foo"].Read()
		indataReplaced := bytes.Replace(indata, []byte("foo"), []byte("bar"), -1)
		task.OutTargets["bar"].WriteTempFile(indataReplaced)
	}
	pathFuncs := map[string]func(*sci.ShellTask) string{
		"bar": func(t *sci.ShellTask) string { return t.InTargets["foo"].GetPath() + ".bar.txt" },
	}
	return NewProcessFromStruct(&FooToBarReplacer{}, execFunc, pathFuncs)
}

// -------------------------------------------
//  New helper methods
// -------------------------------------------

func NewProcessFromStruct(procStruct interface{}, execFunc func(*sci.ShellTask), pathFuncs map[string]func(*sci.ShellTask) string) interface{} {
	// Get in-ports of struct
	inPorts := map[string]chan *sci.FileTarget{}
	outPorts := map[string]chan *sci.FileTarget{}

	procStructVal := r.ValueOf(procStruct).Elem()
	for i := 0; i < procStructVal.NumField(); i++ {
		structFieldName := procStructVal.Type().Field(i).Name
		structFieldType := procStructVal.Type().Field(i).Type
		exampleChan := make(chan *sci.FileTarget)
		if strings.HasPrefix(structFieldName, "In") && structFieldType == r.TypeOf(exampleChan) {
			fmt.Println("In-port:", structFieldName)
			inPorts[strings.ToLower(structFieldName)] = exampleChan // TODO: Change this!
		} else if strings.HasPrefix(structFieldName, "Out") && structFieldType == r.TypeOf(exampleChan) {
			fmt.Println("Out-port:", structFieldName)
			outPorts[strings.ToLower(structFieldName)] = exampleChan // TODO: Change this!
		}
	}
	return procStruct
}

{'fraction_non_alphanumeric': 0.19636746512240064, 'fraction_numerical': 0.0015793629902605948, 'mean_word_length': 3.7322540473225403, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5260157', 'n_tokens_mistral': 1365, 'n_tokens_neox': 1268, 'n_words': 245}