<commit_before>package archive

import (
	"bytes"
	"compress/bzip2"
	"compress/gzip"
	"errors"
	"fmt"
	"github.com/dotcloud/docker/pkg/system"
	"github.com/dotcloud/docker/utils"
	"github.com/dotcloud/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"syscall"
)

type (
	Archive       io.ReadCloser
	ArchiveReader io.Reader
	Compression   int
	TarOptions    struct {
		Includes    []string
		Compression Compression
	}
)

var (
	ErrNotImplemented = errors.New("Function not implemented")
)

const (
	Uncompressed Compression = iota
	Bzip2
	Gzip
	Xz
)

func DetectCompression(source []byte) Compression {
	sourceLen := len(source)
	for compression, m := range map[Compression][]byte{
		Bzip2: {0x42, 0x5A, 0x68},
		Gzip:  {0x1F, 0x8B, 0x08},
		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
	} {
		fail := false
		if len(m) > sourceLen {
			utils.Debugf("Len too short")
			continue
		}
		i := 0
		for _, b := range m {
			if b != source[i] {
				fail = true
				break
			}
			i++
		}
		if !fail {
			return compression
		}
	}
	return Uncompressed
}

func xzDecompress(archive io.Reader) (io.ReadCloser, error) {
	args := []string{"xz", "-d", "-c", "-q"}

	return CmdStream(exec.Command(args[0], args[1:]...), archive)
}

func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
	buf := make([]byte, 10)
	totalN := 0
	for totalN < 10 {
		n, err := archive.Read(buf[totalN:])
		if err != nil {
			if err == io.EOF {
				return nil, fmt.Errorf("Tarball too short")
			}
			return nil, err
		}
		totalN += n
		utils.Debugf("[tar autodetect] n: %d", n)
	}
	compression := DetectCompression(buf)
	wrap := io.MultiReader(bytes.NewReader(buf), archive)

	switch compression {
	case Uncompressed:
		return ioutil.NopCloser(wrap), nil
	case Gzip:
		return gzip.NewReader(wrap)
	case Bzip2:
		return ioutil.NopCloser(bzip2.NewReader(wrap)), nil
	case Xz:
		return xzDecompress(wrap)
	default:
		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
	}
}

func CompressStream(dest io.WriteCloser, compression Compression) (io.WriteCloser, error) {

	switch compression {
	case Uncompressed:
		return utils.NopWriteCloser(dest), nil
	case Gzip:
		return gzip.NewWriter(dest), nil
	case Bzip2, Xz:
		// archive/bzip2 does not support writing, and there is no xz support at all
		// However, this is not a problem as docker only currently generates gzipped tars
		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
	default:
		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
	}
}

func (compression *Compression) Extension() string {
	switch *compression {
	case Uncompressed:
		return "tar"
	case Bzip2:
		return "tar.bz2"<commit_msg>archive: use bufio for compression detection<commit_after>package archive

import (
	"bufio"
	"compress/bzip2"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/dotcloud/docker/pkg/system"
	"github.com/dotcloud/docker/utils"
	"github.com/dotcloud/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
)

type (
	Archive       io.ReadCloser
	ArchiveReader io.Reader
	Compression   int
	TarOptions    struct {
		Includes    []string
		Compression Compression
	}
)

var (
	ErrNotImplemented = errors.New("Function not implemented")
)

const (
	Uncompressed Compression = iota
	Bzip2
	Gzip
	Xz
)

func DetectCompression(source []byte) Compression {
	sourceLen := len(source)
	for compression, m := range map[Compression][]byte{
		Bzip2: {0x42, 0x5A, 0x68},
		Gzip:  {0x1F, 0x8B, 0x08},
		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
	} {
		fail := false
		if len(m) > sourceLen {
			utils.Debugf("Len too short")
			continue
		}
		i := 0
		for _, b := range m {
			if b != source[i] {
				fail = true
				break
			}
			i++
		}
		if !fail {
			return compression
		}
	}
	return Uncompressed
}

func xzDecompress(archive io.Reader) (io.ReadCloser, error) {
	args := []string{"xz", "-d", "-c", "-q"}

	return CmdStream(exec.Command(args[0], args[1:]...), archive)
}

func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
	buf := bufio.NewReader(archive)
	bs, err := buf.Peek(10)
	if err != nil {
		return nil, err
	}
	utils.Debugf("[tar autodetect] n: %v", bs)

	compression := DetectCompression(bs)

	switch compression {
	case Uncompressed:
		return ioutil.NopCloser(buf), nil
	case Gzip:
		return gzip.NewReader(buf)
	case Bzip2:
		return ioutil.NopCloser(bzip2.NewReader(buf)), nil
	case Xz:
		return xzDecompress(buf)
	default:
		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
	}
}

func CompressStream(dest io.WriteCloser, compression Compression) (io.WriteCloser, error) {

	switch compression {
	case Uncompressed:
		return utils.NopWriteCloser(dest), nil
	case Gzip:
		return gzip.NewWriter(dest), nil
	case Bzip2, Xz:
		// archive/bzip2 does not support writing, and there is no xz support at all
		// However, this is not a problem as docker only currently generates gzipped tars
		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
	default:
		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
	}
}

func (compression *Compression) Extension() string {
	switch *compression {
	case Uncompressed:
		return "tar"
	case Bzip2:
		return "tar.bz2"
{'fraction_non_alphanumeric': 0.13439095550692925, 'fraction_numerical': 0.016229029905178702, 'mean_word_length': 3.9325539568345325, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1632199', 'n_tokens_mistral': 2224, 'n_tokens_neox': 2057, 'n_words': 505}