<commit_before>/*
Copyright 2011-2013 <NAME>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
you may obtain a copy of the License at

                http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"kanzi/io"
	"os"
	"strings"
	"strconv"
	"time"
)

const (
	DEFAULT_BUFFER_SIZE = 32768
	WARN_EMPTY_INPUT    = -128
)

type BlockCompressor struct {
	verbose      bool
	silent       bool
	overwrite    bool
	checksum     bool
	inputName    string
	outputName   string
	entropyCodec string
	transform    string
	blockSize    uint
}

func NewBlockCompressor() (*BlockCompressor, error) {
	this := new(BlockCompressor)

	// Define flags
	var help = flag.Bool("help", false, "display the help message")
	var verbose = flag.Bool("verbose", false, "display the block size at each stage (in bytes, floor rounding if fractional)")
	var silent = flag.Bool("silent", false, "silent mode, no output (except warnings and errors)")
	var overwrite = flag.Bool("overwrite", false, "overwrite the output file if it already exists")
	var inputName = flag.String("input", "", "mandatory name of the input file to encode")
	var outputName = flag.String("output", "", "optional name of the output file (defaults to <input.knz>)")
	var blockSize = flag.String("block", "1048576", "size of the input blocks (max 16MB - 4 / min 1KB / default 1MB)")
	var entropy = flag.String("entropy", "Huffman", "entropy codec to use [None|Huffman*|Range|PAQ|FPAQ]")
	var function = flag.String("transform", "Block", "transform to use [None|Block*|Snappy|LZ4|RLT]")
	var cksum = flag.Bool("checksum", false, "enable block checksum")

	// Parse
	flag.Parse()

	if *help == true {
		printOut("-help              : display this message", true)
		printOut("-verbose           : display the block size at each stage (in bytes, floor rounding if fractional)", true)
		printOut("-silent            : silent mode, no output (except warnings and errors)", true)
		printOut("-overwrite         : overwrite the output file if it already exists", true)
		printOut("-input=<filename>  : mandatory name of the input file to encode", true)
		printOut("-output=<filename> : optional name of the output file (defaults to <input.knz>)", true)
		printOut("-block=<size>      : size of the input blocks (max 16MB - 4 / min 1KB / default 1MB)", true)
		printOut("-entropy=          : entropy codec to use [None|Huffman*|Range|PAQ|FPAQ]", true)
		printOut("-transform=        : transform to use [None|Block*|Snappy|LZ4|RLT]", true)
		printOut("-checksum          : enable block checksum", true)
		os.Exit(0)
	}

	if *silent == true && *verbose == true {
		printOut("Warning: both 'silent' and 'verbose' options were selected, ignoring 'verbose'", true)
		*verbose = false
	}

	if len(*inputName) == 0 {
		fmt.Printf("Missing input file name, exiting ...\n")
		os.Exit(io.ERR_MISSING_FILENAME)
	}

	if len(*outputName) == 0 {
		*outputName = *inputName + ".knz"
	}

	this.verbose = *verbose
	this.silent = *silent
	this.overwrite = *overwrite
	this.inputName = *inputName
	this.outputName = *outputName
	strBlockSize := strings.ToUpper(*blockSize)	
	
	// Process K or M suffix
	scale := 1
	
	if strBlockSize[len(strBlockSize)-1] == 'K' {	
		strBlockSize = strBlockSize[0:len(strBlockSize)-1]
		scale = 1024
	} else if strBlockSize[len(strBlockSize)-1] == 'M' {	
		strBlockSize = strBlockSize[0:len(strBlockSize)-1]
		scale = 1024 * 1024
	}

	bSize, err := strconv.Atoi(strBlockSize)

	if err != nil {
		fmt.Printf("Invalid block size provided on command line: %v\n", *blockSize)
		os.Exit(io.ERR_BLOCK_SIZE)
	}

	this.blockSize = uint(scale * bSize)
	this.entropyCodec = strings.ToUpper(*entropy)
	this.transform = strings.ToUpper(*function)
	this.checksum = *cksum
	return this, nil
}

func main() {
	bc, err := NewBlockCompressor()

	if err != nil {
		fmt.Printf("Failed to create block compressor: %v\n", err)
		os.Exit(io.ERR_CREATE_COMPRESSOR)
	}

	defer func() {
		if r := recover(); r != nil {
			fmt.Printf("An unexpected error occured during compression: %v\n", r.(error))
			os.Exit(io.ERR_UNKNOWN)
		}
	}()

	code, _ := bc.call()
	os.Exit(code)
}

// Return exit code, number of bits written
func (this *BlockCompressor) call() (int, uint64) {
	var msg string
	printOut("Input file name set to '"+this.inputName+"'", this.verbose)
	printOut("Output file name set to '"+this.outputName+"'", this.verbose)
	msg = fmt.Sprintf("Block size set to %d bytes", this.blockSize)
	printOut(msg, this.verbose)
	msg = fmt.Sprintf("Debug set to %t", this.verbose)
	printOut(msg, this.verbose)
	msg = fmt.Sprintf("Overwrite set to %t", this.overwrite)
	printOut(msg, this.verbose)
	msg = fmt.Sprintf("Checksum set to %t", this.checksum)
	printOut(msg, this.verbose)
	w1 := "no"

	if this.transform != "NONE" {
		w1 = this.transform
	}

	msg = fmt.Sprintf("Using %s transform (stage 1)", w1)
	printOut(msg, this.verbose)
	w2 := "no"

	if this.entropyCodec != "NONE" {
		w2 = this.entropyCodec
	}

	msg = fmt.Sprintf("Using %s entropy codec (stage 2)", w2)
	printOut(msg, this.verbose)
	written := uint64(0)
	output, err := os.OpenFile(this.outputName, os.O_RDWR, 666)

	if err == nil {
		// File exists
		if this.overwrite == false {
			fmt.Print("The output file exists and the 'overwrite' command ")
			fmt.Println("line option has not been provided")
			output.Close()
			return io.ERR_OVERWRITE_FILE, written
		}
	} else {
		// File does not exist, create
		output, err = os.Create(this.outputName)

		if err != nil {
			fmt.Printf("Cannot open output file '%v' for writing: %v\n", this.outputName, err)
			return io.ERR_CREATE_FILE, written
		}
	}

	defer output.Close()
	verboseWriter := os.Stdout

	if this.verbose == false {
		verboseWriter = nil
	}

	cos, err := io.NewCompressedOutputStream(this.entropyCodec, this.transform,
		output, this.blockSize, this.checksum, verboseWriter)

	if err != nil {
		if ioerr, isIOErr := err.(io.IOError); isIOErr == true {
			fmt.Printf("%s\n", ioerr.Error())
			return ioerr.ErrorCode(), written
		} else {
			fmt.Printf("Cannot create compressed stream: %s\n", err.Error())
			return io.ERR_CREATE_COMPRESSOR, written
		}
	}

	defer cos.Close()
	input, err := os.Open(this.inputName)<commit_msg>Re-create the output file if it exists (and the overwrite option has been provided). Previously, an existing file would get truncated which confused the decompressor when reading the size (potential incorrect rounding of read bytes).<commit_after>/*
Copyright 2011-2013 <NAME>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
you may obtain a copy of the License at

                http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"kanzi/io"
	"os"
	"strings"
	"strconv"
	"time"
)

const (
	DEFAULT_BUFFER_SIZE = 32768
	WARN_EMPTY_INPUT    = -128
)

type BlockCompressor struct {
	verbose      bool
	silent       bool
	overwrite    bool
	checksum     bool
	inputName    string
	outputName   string
	entropyCodec string
	transform    string
	blockSize    uint
}

func NewBlockCompressor() (*BlockCompressor, error) {
	this := new(BlockCompressor)

	// Define flags
	var help = flag.Bool("help", false, "display the help message")
	var verbose = flag.Bool("verbose", false, "display the block size at each stage (in bytes, floor rounding if fractional)")
	var silent = flag.Bool("silent", false, "silent mode, no output (except warnings and errors)")
	var overwrite = flag.Bool("overwrite", false, "overwrite the output file if it already exists")
	var inputName = flag.String("input", "", "mandatory name of the input file to encode")
	var outputName = flag.String("output", "", "optional name of the output file (defaults to <input.knz>)")
	var blockSize = flag.String("block", "1048576", "size of the input blocks (max 16MB - 4 / min 1KB / default 1MB)")
	var entropy = flag.String("entropy", "Huffman", "entropy codec to use [None|Huffman*|Range|PAQ|FPAQ]")
	var function = flag.String("transform", "Block", "transform to use [None|Block*|Snappy|LZ4|RLT]")
	var cksum = flag.Bool("checksum", false, "enable block checksum")

	// Parse
	flag.Parse()

	if *help == true {
		printOut("-help              : display this message", true)
		printOut("-verbose           : display the block size at each stage (in bytes, floor rounding if fractional)", true)
		printOut("-silent            : silent mode, no output (except warnings and errors)", true)
		printOut("-overwrite         : overwrite the output file if it already exists", true)
		printOut("-input=<filename>  : mandatory name of the input file to encode", true)
		printOut("-output=<filename> : optional name of the output file (defaults to <input.knz>)", true)
		printOut("-block=<size>      : size of the input blocks (max 16MB - 4 / min 1KB / default 1MB)", true)
		printOut("-entropy=          : entropy codec to use [None|Huffman*|Range|PAQ|FPAQ]", true)
		printOut("-transform=        : transform to use [None|Block*|Snappy|LZ4|RLT]", true)
		printOut("-checksum          : enable block checksum", true)
		os.Exit(0)
	}

	if *silent == true && *verbose == true {
		printOut("Warning: both 'silent' and 'verbose' options were selected, ignoring 'verbose'", true)
		*verbose = false
	}

	if len(*inputName) == 0 {
		fmt.Printf("Missing input file name, exiting ...\n")
		os.Exit(io.ERR_MISSING_FILENAME)
	}

	if len(*outputName) == 0 {
		*outputName = *inputName + ".knz"
	}

	this.verbose = *verbose
	this.silent = *silent
	this.overwrite = *overwrite
	this.inputName = *inputName
	this.outputName = *outputName
	strBlockSize := strings.ToUpper(*blockSize)	
	
	// Process K or M suffix
	scale := 1
	
	if strBlockSize[len(strBlockSize)-1] == 'K' {	
		strBlockSize = strBlockSize[0:len(strBlockSize)-1]
		scale = 1024
	} else if strBlockSize[len(strBlockSize)-1] == 'M' {	
		strBlockSize = strBlockSize[0:len(strBlockSize)-1]
		scale = 1024 * 1024
	}

	bSize, err := strconv.Atoi(strBlockSize)

	if err != nil {
		fmt.Printf("Invalid block size provided on command line: %v\n", *blockSize)
		os.Exit(io.ERR_BLOCK_SIZE)
	}

	this.blockSize = uint(scale * bSize)
	this.entropyCodec = strings.ToUpper(*entropy)
	this.transform = strings.ToUpper(*function)
	this.checksum = *cksum
	return this, nil
}

func main() {
	bc, err := NewBlockCompressor()

	if err != nil {
		fmt.Printf("Failed to create block compressor: %v\n", err)
		os.Exit(io.ERR_CREATE_COMPRESSOR)
	}

	defer func() {
		if r := recover(); r != nil {
			fmt.Printf("An unexpected error occured during compression: %v\n", r.(error))
			os.Exit(io.ERR_UNKNOWN)
		}
	}()

	code, _ := bc.call()
	os.Exit(code)
}

// Return exit code, number of bits written
func (this *BlockCompressor) call() (int, uint64) {
	var msg string
	printOut("Input file name set to '"+this.inputName+"'", this.verbose)
	printOut("Output file name set to '"+this.outputName+"'", this.verbose)
	msg = fmt.Sprintf("Block size set to %d bytes", this.blockSize)
	printOut(msg, this.verbose)
	msg = fmt.Sprintf("Debug set to %t", this.verbose)
	printOut(msg, this.verbose)
	msg = fmt.Sprintf("Overwrite set to %t", this.overwrite)
	printOut(msg, this.verbose)
	msg = fmt.Sprintf("Checksum set to %t", this.checksum)
	printOut(msg, this.verbose)
	w1 := "no"

	if this.transform != "NONE" {
		w1 = this.transform
	}

	msg = fmt.Sprintf("Using %s transform (stage 1)", w1)
	printOut(msg, this.verbose)
	w2 := "no"

	if this.entropyCodec != "NONE" {
		w2 = this.entropyCodec
	}

	msg = fmt.Sprintf("Using %s entropy codec (stage 2)", w2)
	printOut(msg, this.verbose)
	written := uint64(0)
	output, err := os.OpenFile(this.outputName, os.O_RDWR, 666)

	if err == nil {
		// File exists
		output.Close()
		
		if this.overwrite == false {
			fmt.Print("The output file exists and the 'overwrite' command ")
			fmt.Println("line option has not been provided")			
			return io.ERR_OVERWRITE_FILE, written
		}
	}

	output, err = os.Create(this.outputName)

	if err != nil {
		fmt.Printf("Cannot open output file '%v' for writing: %v\n", this.outputName, err)
		return io.ERR_CREATE_FILE, written
	}

	defer output.Close()
	verboseWriter := os.Stdout

	if this.verbose == false {
		verboseWriter = nil
	}

	cos, err := io.NewCompressedOutputStream(this.entropyCodec, this.transform,
		output, this.blockSize, this.checksum, verboseWriter)

	if err != nil {
		if ioerr, isIOErr := err.(io.IOError); isIOErr == true {
			fmt.Printf("%s\n", ioerr.Error())
			return ioerr.ErrorCode(), written
		} else {
			fmt.Printf("Cannot create compressed stream: %s\n", err.Error())
			return io.ERR_CREATE_COMPRESSOR, written
		}
	}

	defer cos.Close()
	input, err := os.Open(this.inputName)
{'fraction_non_alphanumeric': 0.13058753176857527, 'fraction_numerical': 0.01151143668709822, 'mean_word_length': 3.8846294267981016, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3934292', 'n_tokens_mistral': 4693, 'n_tokens_neox': 4398, 'n_words': 1367}