func (ip *FileIP) createDirs(baseDir string) {
	ipDir := ip.TempDir()
	if baseDir != "" {
		ipDir = baseDir + "/" + ip.TempDir()
	}
	if ip.doStream {
		ipDir = filepath.Dir(ip.FifoPath())
	}
	err := os.MkdirAll(ipDir, 0777)
	if err != nil {
		ip.Failf("Could not create directory: (%s): %s\n", ipDir, err)
	}
}
CreateDirs creates all directories needed to enable writing the IP to its
path (or temporary-path). If baseDir is provided, it will be prepended
before all the IPs own temp path. This is to allow components to create their
own temporary directory, to create the tasks in.
package scipipe
import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"
)
type IP interface {
	ID() string
	FinalizePath()
}
type BaseIP struct {
	path      string
	id        string
	auditInfo *AuditInfo
}
func NewBaseIP(path string) *BaseIP {
	return &BaseIP{
		path: path,
		id:   randSeqLC(20),
	}
}
func (ip *BaseIP) ID() string {
	return ip.id
}
type FileIP struct {
	*BaseIP
	buffer    *bytes.Buffer
	doStream  bool
	lock      *sync.Mutex
	SubStream *InPort
}
func NewFileIP(path string) (*FileIP, error) {
	isValid, err := pathIsValid(path)
	if err != nil {
		return nil, err
	}
	if !isValid {
		return nil, errors.New(fmt.Sprintf(`Could not create new FileIP with filename "%s". File is invalid (perhaps contains invalid characters)`, path))
	}
	ip := &FileIP{
		BaseIP:    NewBaseIP(path),
		lock:      &sync.Mutex{},
		SubStream: NewInPort("in_substream"),
	}
	if ip.Exists() {
		ip.AuditInfo()  
	}
	return ip, nil
}
func pathIsValid(path string) (bool, error) {
	expr := `^[0-9A-Za-z\/\.\-_]+$`
	ptn, err := regexp.Compile(expr)
	if err != nil {
		return false, errors.New("Could not compile regex")
	}
	return ptn.MatchString(path), nil
}
func (ip *FileIP) Path() string {
	return ip.path
}
func (ip *FileIP) TempDir() string {
	return filepath.Dir(ip.TempPath())
}
func (ip *FileIP) TempPath() string {
	path := replaceParentDirsWithPlaceholder(ip.path)
	if path[0] == '/' {
		return FSRootPlaceHolder + path
	}
	return path
}
const FSRootPlaceHolder = "__fsroot__"
func (ip *FileIP) FifoPath() string {
	return ip.path + ".fifo"
}
func (ip *FileIP) Size() int64 {
	fi, err := os.Stat(ip.path)
	Check(err)
	return fi.Size()
}
func (ip *FileIP) Exists() bool {
	exists := false
	ip.lock.Lock()
	if _, err := os.Stat(ip.Path()); err == nil {
		exists = true
	}
	ip.lock.Unlock()
	return exists
}
func (ip *FileIP) TempFileExists() bool {
	tempFileExists := false
	ip.lock.Lock()
	if _, err := os.Stat(ip.TempPath()); err == nil {
		tempFileExists = true
	}
	ip.lock.Unlock()
	return tempFileExists
}
func (ip *FileIP) FifoFileExists() bool {
	fifoFileExists := false
	ip.lock.Lock()
	if _, err := os.Stat(ip.FifoPath()); err == nil {
		fifoFileExists = true
	}
	ip.lock.Unlock()
	return fifoFileExists
}
func (ip *FileIP) String() string {
	return ip.Path()
}
func (ip *FileIP) Open() *os.File {
	f, err := os.Open(ip.Path())
	CheckWithMsg(err, "Could not open file: "+ip.Path())
	return f
}
func (ip *FileIP) OpenTemp() *os.File {
	f, err := os.Open(ip.TempPath())
	CheckWithMsg(err, "Could not open temp file: "+ip.TempPath())
	return f
}
func (ip *FileIP) CreateFifo() {
	ip.createDirs("")
	ip.lock.Lock()
	cmd := "mkfifo " + ip.FifoPath()
	Debug.Println("Now creating FIFO with command:", cmd)
	if _, err := os.Stat(ip.FifoPath()); err == nil {
		Warning.Printf("[FileIP:%s] FIFO already exists, so not creating a new one: %s", ip.Path(), ip.FifoPath())
	} else {
		_, err := exec.Command("bash", "-c", cmd).Output()
		CheckWithMsg(err, "Could not execute command: "+cmd)
	}
	ip.lock.Unlock()
}
func (ip *FileIP) RemoveFifo() {
	ip.lock.Lock()
	output, err := exec.Command("bash", "-c", "rm "+ip.FifoPath()).Output()
	CheckWithMsg(err, "Could not delete fifo file: "+ip.FifoPath())
	Debug.Println("Removed FIFO output: ", output)
	ip.lock.Unlock()
}
func (ip *FileIP) Read() []byte {
	dat, err := ioutil.ReadFile(ip.Path())
	CheckWithMsg(err, "Could not open file for reading: "+ip.Path())
	return dat
}
func (ip *FileIP) Write(dat []byte) {
	ip.createDirs("")
	err := ioutil.WriteFile(ip.TempPath(), dat, 0644)
	CheckWithMsg(err, "Could not write to temp file: "+ip.TempPath())
}
const (
	finalizePathMaxTries      = 3
	finalizePathBackoffFactor = 4
)
func (ip *FileIP) FinalizePath() {
	Debug.Println("FileIP: Finalizing path ", ip.TempPath(), "->", ip.Path())
	doneFinalizingPath := false
	tries := 0
	sleepDurationSec := 1
	for !doneFinalizingPath {
		if ip.TempFileExists() {
			ip.lock.Lock()
			tempPaths, err := filepath.Glob(ip.TempDir() + "/*")
			CheckWithMsg(err, "Could not blog directory: "+ip.TempDir())
			for _, tempPath := range tempPaths {
				origDir := filepath.Dir(ip.TempDir())
				origFileName := filepath.Base(tempPath)
				err := os.Rename(tempPath, origDir+"/"+origFileName)
				CheckWithMsg(err, "Could not rename file: "+ip.TempPath())
			}
			err = os.Remove(ip.TempDir())
			CheckWithMsg(err, "Could not remove temp dir: "+ip.TempDir())
			ip.lock.Unlock()
			doneFinalizingPath = true
			Debug.Println("FileIP: Done finalizing path ", ip.TempPath(), "->", ip.Path())
		} else {
			if tries >= finalizePathMaxTries {
				ip.Failf("Failed to find .tmp file after %d tries, so shutting down: %s\nNote: If this problem persists, it could be a problem with your workflow, that the configured output filename in scipipe doesn't match what is written by the tool.", finalizePathMaxTries, ip.TempPath())
			}
			Warning.Printf("[FileIP:%s] Expected .tmp file missing: %s\nSleeping for %d seconds before checking again ...\n", ip.Path(), ip.TempPath(), sleepDurationSec)
			time.Sleep(time.Duration(sleepDurationSec) * time.Second)
			sleepDurationSec *= finalizePathBackoffFactor
			tries++
		}
	}
}
func (ip *FileIP) Param(key string) string {
	val, ok := ip.AuditInfo().Params[key]
	if !ok {
		ip.Failf("Could not find parameter %s", key)
	}
	return val
}
func (ip *FileIP) Tag(k string) string {
	v, ok := ip.AuditInfo().Tags[k]
	if !ok {
		Warning.Printf("[FileIP:%s] No such tag: (%s)\n", ip.Path(), k)
		return ""
	}
	return v
}
func (ip *FileIP) Tags() map[string]string {
	return ip.AuditInfo().Tags
}
func (ip *FileIP) AddTag(k string, v string) {
	ai := ip.AuditInfo()
	if ai.Tags[k] != "" && ai.Tags[k] != v {
		ip.Failf("Can not add value (%s) to existing tag (%s) with different value (%s)", v, k, ai.Tags[k])
	}
	ai.Tags[k] = v
}
func (ip *FileIP) AddTags(tags map[string]string) {
	for k, v := range tags {
		ip.AddTag(k, v)
	}
}
func (ip *FileIP) AuditFilePath() string {
	return ip.Path() + ".audit.json"
}
func (ip *FileIP) SetAuditInfo(ai *AuditInfo) {
	ip.lock.Lock()
	ip.auditInfo = ai
	ip.lock.Unlock()
}
func (ip *FileIP) WriteAuditLogToFile() {
	auditInfo := ip.AuditInfo()
	auditInfoJSON, jsonErr := json.MarshalIndent(auditInfo, "", "    ")
	CheckWithMsg(jsonErr, "Could not marshall JSON")
	ip.createDirs("")
	writeErr := ioutil.WriteFile(ip.AuditFilePath(), auditInfoJSON, 0644)
	CheckWithMsg(writeErr, "Could not write audit file: "+ip.Path())
}
func (ip *FileIP) AuditInfo() *AuditInfo {
	defer ip.lock.Unlock()
	ip.lock.Lock()
	if ip.auditInfo == nil {
		ip.auditInfo = UnmarshalAuditInfoJSONFile(ip.AuditFilePath())
	}
	return ip.auditInfo
}
func UnmarshalAuditInfoJSONFile(fileName string) (auditInfo *AuditInfo) {
	auditInfo = NewAuditInfo()
	auditFileData, readFileErr := ioutil.ReadFile(fileName)
	if readFileErr != nil {
		if os.IsNotExist(readFileErr) {
			Info.Printf("Audit file not found, so not unmarshalling: %s\n", fileName)
		} else {
			Failf("Could not read audit file, which does exist: %s", fileName)
		}
	} else {
		unmarshalErr := json.Unmarshal(auditFileData, auditInfo)
		CheckWithMsg(unmarshalErr, "Could not unmarshal audit log file content: "+fileName)
	}
	return auditInfo
}
func (ip *FileIP) UnMarshalJSON(v interface{}) {
	d := ip.Read()
	err := json.Unmarshal(d, v)
	CheckWithMsg(err, "Could not unmarshal content of file: "+ip.Path())
}
func (ip *FileIP) Failf(msg string, parts ...interface{}) {
	ip.Fail(fmt.Sprintf(msg+"\n", parts...))
}
func (ip *FileIP) Fail(msg interface{}) {
	Failf("[FileIP:%s]: %s", ip.Path(), msg)
}