<commit_before>package main

import (
	"io/ioutil"
	"log"
	"os"
	"strconv"

	"github.com/codegangsta/cli"
	"github.com/minio-io/minio/erasure"
)

func main() {
	app := cli.NewApp()
	app.Name = "minio-encode"
	app.Usage = "erasure encode a byte stream"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "input,i",
			Value: "",
			Usage: "Input file",
		},
		cli.StringFlag{
			Name:  "output,o",
			Value: "",
			Usage: "Output file",
		},
		cli.IntFlag{
			Name:  "k",
			Value: 10,
			Usage: "k value of encoder parameters",
		},
		cli.IntFlag{
			Name:  "m",
			Value: 5,
			Usage: "m value of encoder parameters",
		},
	}
	app.Action = func(c *cli.Context) {
		// check if minio-encode called without parameters
		if len(c.Args()) == 1 {
			cli.ShowAppHelp(c)
		}

		// get input path
		if c.String("input") == "" {
			log.Fatal("No input specified")
		}
		inputFilePath := c.String("input")

		// get output path
		outputFilePath := inputFilePath
		if c.String("output") != "" {
			outputFilePath = c.String("output")
		}

		k := c.Int("k")
		m := c.Int("m")

		// get chunks
		chunks := make([][]byte, k+m)
		for i := 0; i < k+m; i++ {
			var err error
			chunks[i], err = ioutil.ReadFile(inputFilePath + "." + strconv.Itoa(i))
			if err != nil {
				log.Fatal(err)
			}
		}

		// get length
		lengthBytes, err := ioutil.ReadFile(inputFilePath + ".length")
		if err != nil {
			log.Fatal(err)
		}
		lengthString := string(lengthBytes)
		length, err := strconv.Atoi(lengthString)
		if err != nil {
			log.Fatal(err)
		}

		// set up encoder
		erasureParameters, _ := erasure.ValidateParams(k, m, 8, erasure.CAUCHY)
		encoder := erasure.NewEncoder(erasureParameters)

		// decode data
		decodedData, err := encoder.Decode(chunks, length)
		if err != nil {
			log.Fatal(err)
		}

		// write decode data out
		ioutil.WriteFile(outputFilePath, decodedData, 0600)
	}
	app.Run(os.Args)
}
<commit_msg>Decode on chunks missing should build and continue<commit_after>package main

import (
	"io/ioutil"
	"log"
	"os"
	"strconv"

	"github.com/codegangsta/cli"
	"github.com/minio-io/minio/erasure"
)

func main() {
	app := cli.NewApp()
	app.Name = "minio-decode"
	app.Usage = "erasure decode a byte stream"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "input,i",
			Value: "",
			Usage: "Input file",
		},
		cli.StringFlag{
			Name:  "output,o",
			Value: "",
			Usage: "Output file",
		},
		cli.IntFlag{
			Name:  "k",
			Value: 10,
			Usage: "k value of encoder parameters",
		},
		cli.IntFlag{
			Name:  "m",
			Value: 5,
			Usage: "m value of encoder parameters",
		},
	}
	app.Action = func(c *cli.Context) {
		// check if minio-encode called without parameters
		if len(c.Args()) == 1 {
			cli.ShowAppHelp(c)
		}

		// get input path
		if c.String("input") == "" {
			log.Fatal("No input specified")
		}
		inputFilePath := c.String("input")

		// get output path
		outputFilePath := inputFilePath
		if c.String("output") != "" {
			outputFilePath = c.String("output")
		}

		k := c.Int("k")
		m := c.Int("m")

		// get chunks
		chunks := make([][]byte, k+m)
		for i := 0; i < k+m; i++ {
			var err error
			chunks[i], err = ioutil.ReadFile(inputFilePath + "." + strconv.Itoa(i))
			if err != nil {
				continue
			}
		}

		// get length
		lengthBytes, err := ioutil.ReadFile(inputFilePath + ".length")
		if err != nil {
			log.Fatal(err)
		}
		lengthString := string(lengthBytes)
		length, err := strconv.Atoi(lengthString)
		if err != nil {
			log.Fatal(err)
		}

		// set up encoder
		erasureParameters, _ := erasure.ValidateParams(k, m, 8, erasure.CAUCHY)
		encoder := erasure.NewEncoder(erasureParameters)

		// decode data
		decodedData, err := encoder.Decode(chunks, length)
		if err != nil {
			log.Fatal(err)
		}

		// write decode data out
		ioutil.WriteFile(outputFilePath, decodedData, 0600)
	}
	app.Run(os.Args)
}

{'fraction_non_alphanumeric': 0.16610169491525423, 'fraction_numerical': 0.005215123859191656, 'mean_word_length': 2.950566426364573, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3480826', 'n_tokens_mistral': 1686, 'n_tokens_neox': 1452, 'n_words': 335}