<commit_before>package scipipe

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"sync"
	"time"
)

// ======= FileIP ========

// FileIP (Short for "Information Packet" in Flow-Based Programming terminology)
// contains information and helper methods for a physical file on a normal disk.
type FileIP struct {
	path      string
	buffer    *bytes.Buffer
	doStream  bool
	lock      *sync.Mutex
	auditInfo *AuditInfo
	SubStream *InPort
}

// NewFileIP creates a new FileIP
func NewFileIP(path string) *FileIP {
	ip := new(FileIP)
	ip.path = path
	ip.lock = new(sync.Mutex)
	ip.SubStream = NewInPort("in_substream")
	//Don't init buffer if not needed?
	//buf := make([]byte, 0, 128)
	//ip.buffer = bytes.NewBuffer(buf)
	return ip
}

// Path returns the (final) path of the physical file
func (ip *FileIP) Path() string {
	return ip.path
}

// TempPath returns the temporary path of the physical file<commit_msg>Use &-syntax in IP factory method<commit_after>package scipipe

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"sync"
	"time"
)

// ======= FileIP ========

// FileIP (Short for "Information Packet" in Flow-Based Programming terminology)
// contains information and helper methods for a physical file on a normal disk.
type FileIP struct {
	path      string
	buffer    *bytes.Buffer
	doStream  bool
	lock      *sync.Mutex
	auditInfo *AuditInfo
	SubStream *InPort
}

// NewFileIP creates a new FileIP
func NewFileIP(path string) *FileIP {
	ip := &FileIP{
		path:      path,
		lock:      &sync.Mutex{},
		SubStream: NewInPort("in_substream"),
	}
	//Don't init buffer if not needed?
	//buf := make([]byte, 0, 128)
	//ip.buffer = bytes.NewBuffer(buf)
	return ip
}

// Path returns the (final) path of the physical file
func (ip *FileIP) Path() string {
	return ip.path
}

// TempPath returns the temporary path of the physical file
{'fraction_non_alphanumeric': 0.12620320855614972, 'fraction_numerical': 0.0042780748663101605, 'mean_word_length': 3.5193236714975846, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1701572', 'n_tokens_mistral': 660, 'n_tokens_neox': 639, 'n_words': 212}