<commit_before>package main

import (
	"bufio"
	"fmt"
	"net"
	"os"
	"path"

	"github.com/gesquive/cli"
	"github.com/gesquive/rip/format"
	"github.com/spf13/cobra"
)

var cfgFile string
var displayVersion string

var logDebug bool
var showVersion bool

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:       "rip [flags] <host>[:<port>] <tcp|udp> <file_path> [<file_path>...]",
	Short:     "Sends a text file line by line to a remote host/port",
	Long:      `Sends a text file line by line to a remote host/port.`,
	ValidArgs: []string{"host:port", "tcp|udp", "file_path"},
	Run:       run,
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute(version string) {
	displayVersion = version
	RootCmd.SetHelpTemplate(fmt.Sprintf("%s\nVersion:\n  github.com/gesquive/%s\n",
		RootCmd.HelpTemplate(), displayVersion))
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	RootCmd.PersistentFlags().BoolVarP(&logDebug, "debug", "D", false,
		"Write debug messages to console")
	RootCmd.PersistentFlags().BoolVarP(&showVersion, "version", "V", false,
		"Show the version and exit")

	RootCmd.PersistentFlags().MarkHidden("debug")
}

func initConfig() {
	if logDebug {
		cli.SetPrintLevel(cli.LevelDebug)
	}
	if showVersion {
		cli.Info(displayVersion)
		os.Exit(0)
	}
	cli.Debug("Running with debug turned on")
}

func run(cmd *cobra.Command, args []string) {
	if len(args) < 2 {
		cmd.Usage()
		os.Exit(1)
	}

	address := args[0]
	protocol := args[1]
	files := args[2:]
	cli.Info("Sending data to %s://%s", protocol, address)

	// Detect if user is piping in text
	fileInput, err := os.Stdin.Stat()
	if err != nil {
		cli.Error(err.Error())
		os.Exit(2)
	}

	pipeFound := fileInput.Mode()&os.ModeNamedPipe != 0
	if pipeFound {
		cli.Debug("Pipe input detected, sending")
		err := sendTextFile(os.Stdin, protocol, address)
		if err != nil {
			cli.Error("Failed to send piped data")
			cli.Error(err.Error())
		}
	}

	errCount := 0
	for _, f := range files {
		textFile, err := os.Open(f)
		defer textFile.Close()
		if err != nil {
			cli.Errorf("Failed to send '%s'\n", f)
			cli.Error(err.Error())
			errCount++
			continue
		}
		err = sendTextFile(textFile, protocol, address)
		if err != nil {
			cli.Errorf("Failed to send '%s' to %s://%s\n", f, protocol, address)
			cli.Error(err.Error())
			errCount++
		}
	}

	if !pipeFound && len(files) == 0 {
		cli.Warn("No data was piped into or specified on the command line.\n")
		cmd.Usage()
		os.Exit(1)
	} else if errCount == 0 {
		cli.Info("All files successfully sent.")
	} else {
		cli.Warn("There were some errors while sending files.")
	}
}

func sendTextFile(textFile *os.File, network string, address string) (err error) {
	fileInfo, err := textFile.Stat()
	if err != nil {
		return err
	}
	bytesRead := uint64(0)
	linesRead := uint64(0)
	totalSize := uint64(fileInfo.Size())
	fileName := path.Base(textFile.Name())

	scanner := bufio.NewScanner(textFile)
	scanner.Split(bufio.ScanLines)

	destPort, err := net.Dial(network, address)
	if err != nil {
		return err
	}

	for scanner.Scan() {
		line := scanner.Text()
		linesRead++
		bytesRead += uint64(len(line)) + 1
		_, err = fmt.Fprint(destPort, line)
		if err != nil {
			cli.Errorf("\r")
			return err
		}
		if linesRead%1000 == 0 {
			cli.Infof("\rtransfer: %s %6dpkts %s", fileName, linesRead, format.Progress(bytesRead, totalSize))
		}
	}
	cli.Info(cli.Green("\rtransfer: Successfully sent '%s' (%d packets)", fileName, linesRead))
	return err
}<commit_msg>move status printout to seperate goroutine<commit_after>package main

import (
	"bufio"
	"fmt"
	"net"
	"os"
	"path"
	"time"

	"github.com/gesquive/cli"
	"github.com/gesquive/rip/format"
	"github.com/spf13/cobra"
)

var cfgFile string
var displayVersion string

var logDebug bool
var showVersion bool

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:       "rip [flags] <host>[:<port>] <tcp|udp> <file_path> [<file_path>...]",
	Short:     "Sends a text file line by line to a remote host/port",
	Long:      `Sends a text file line by line to a remote host/port.`,
	ValidArgs: []string{"host:port", "tcp|udp", "file_path"},
	Run:       run,
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute(version string) {
	displayVersion = version
	RootCmd.SetHelpTemplate(fmt.Sprintf("%s\nVersion:\n  github.com/gesquive/%s\n",
		RootCmd.HelpTemplate(), displayVersion))
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	RootCmd.PersistentFlags().BoolVarP(&logDebug, "debug", "D", false,
		"Write debug messages to console")
	RootCmd.PersistentFlags().BoolVarP(&showVersion, "version", "V", false,
		"Show the version and exit")

	RootCmd.PersistentFlags().MarkHidden("debug")
}

func initConfig() {
	if logDebug {
		cli.SetPrintLevel(cli.LevelDebug)
	}
	if showVersion {
		cli.Info(displayVersion)
		os.Exit(0)
	}
	cli.Debug("Running with debug turned on")
}

func run(cmd *cobra.Command, args []string) {
	if len(args) < 2 {
		cmd.Usage()
		os.Exit(1)
	}

	address := args[0]
	protocol := args[1]
	files := args[2:]
	cli.Info("Sending data to %s://%s", protocol, address)

	// Detect if user is piping in text
	fileInput, err := os.Stdin.Stat()
	if err != nil {
		cli.Error(err.Error())
		os.Exit(2)
	}

	pipeFound := fileInput.Mode()&os.ModeNamedPipe != 0
	if pipeFound {
		cli.Debug("Pipe input detected, sending")
		err := sendTextFile(os.Stdin, protocol, address)
		if err != nil {
			cli.Error("Failed to send piped data")
			cli.Error(err.Error())
		}
	}

	errCount := 0
	for _, f := range files {
		textFile, err := os.Open(f)
		defer textFile.Close()
		if err != nil {
			cli.Errorf("Failed to send '%s'\n", f)
			cli.Error(err.Error())
			errCount++
			continue
		}
		err = sendTextFile(textFile, protocol, address)
		if err != nil {
			cli.Errorf("Failed to send '%s' to %s://%s\n", f, protocol, address)
			cli.Error(err.Error())
			errCount++
		}
	}

	if !pipeFound && len(files) == 0 {
		cli.Warn("No data was piped into or specified on the command line.\n")
		cmd.Usage()
		os.Exit(1)
	} else if errCount == 0 {
		cli.Info("All files successfully sent.")
	} else {
		cli.Warn("There were some errors while sending files.")
	}
}

func sendTextFile(textFile *os.File, network string, address string) (err error) {
	fileInfo, err := textFile.Stat()
	if err != nil {
		return err
	}
	bytesRead := uint64(0)
	linesRead := uint64(0)
	totalSize := uint64(fileInfo.Size())
	fileName := path.Base(textFile.Name())

	scanner := bufio.NewScanner(textFile)
	scanner.Split(bufio.ScanLines)

	destPort, err := net.Dial(network, address)
	if err != nil {
		return err
	}

	ticker := time.NewTicker(100 * time.Millisecond)
	stats := make(chan struct{})

	go func() {
		for {
			select {
			case <-ticker.C:
				cli.Infof("\rtransfer: %s %6dpkts %s", fileName, linesRead, format.Progress(bytesRead, totalSize))
			case <-stats:
				ticker.Stop()
				return
			}
		}
	}()

	start := time.Now()
	for scanner.Scan() {
		line := scanner.Text()
		linesRead++
		bytesRead += uint64(len(line)) + 1
		_, err = fmt.Fprint(destPort, line)
		if err != nil {
			cli.Errorf("\r")
			return err
		}
	}
	end := time.Now()

	close(stats)
	cli.Info(cli.Green("\rtransfer: Successfully sent '%s' (%d packets in %s secs) %.2f pkt/sec",
		fileName, linesRead, end.Sub(start).String(), float64(linesRead)/end.Sub(start).Seconds()))
	return err
}
{'fraction_non_alphanumeric': 0.15523650813998205, 'fraction_numerical': 0.00833226509421869, 'mean_word_length': 3.737097753491196, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2568956', 'n_tokens_mistral': 3015, 'n_tokens_neox': 2815, 'n_words': 747}