<commit_before>package mimetype

import (
	"bufio"
	"io"
	"log"
	"strings"

	"github.com/rakyll/magicmime"
)

//go:generate counterfeiter . Decoder

type Decoder interface {
	TypeByBuffer([]byte) (string, error)
}

var decoder *magicmime.Decoder
var archiveMimetypes = []string{
	"application/x-gzip",
	"application/gzip",
	"application/x-tar",
	"application/zip",
}

func init() {
	var err error
	decoder, err = magicmime.NewDecoder(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
	if err != nil {
		log.Fatalf("failed to make new decoder: %s", err.Error())
	}
}

func IsArchive(r *bufio.Reader) (string, bool) {
	bs, err := r.Peek(512)
	if err != nil && err != io.EOF {
		log.Fatalf("failed to peek: %#v, bs: %s", err, string(bs))
	}

	if len(bs) == 0 {
		return "", false
	}

	mime, err := decoder.TypeByBuffer(bs)
	if err != nil {
		log.Fatalf("failed to get mimetype: %s", err.Error())
	}

	for _, mimetype := range archiveMimetypes {
		if strings.HasPrefix(mime, mimetype) {
			return mimetype, true
		}
	}

	return mime, false
}
<commit_msg>IsArchive returns false for nil reader<commit_after>package mimetype

import (
	"bufio"
	"io"
	"log"
	"strings"

	"github.com/rakyll/magicmime"
)

//go:generate counterfeiter . Decoder

type Decoder interface {
	TypeByBuffer([]byte) (string, error)
}

var decoder *magicmime.Decoder
var archiveMimetypes = []string{
	"application/x-gzip",
	"application/gzip",
	"application/x-tar",
	"application/zip",
}

func init() {
	var err error
	decoder, err = magicmime.NewDecoder(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
	if err != nil {
		log.Fatalf("failed to make new decoder: %s", err.Error())
	}
}

func IsArchive(r *bufio.Reader) (string, bool) {
	if r == nil {
		return "", false
	}

	bs, err := r.Peek(512)
	if err != nil && err != io.EOF {
		log.Fatalf("failed to peek: %#v, bs: %s", err, string(bs))
	}

	if len(bs) == 0 {
		return "", false
	}

	mime, err := decoder.TypeByBuffer(bs)
	if err != nil {
		log.Fatalf("failed to get mimetype: %s", err.Error())
	}

	for _, mimetype := range archiveMimetypes {
		if strings.HasPrefix(mime, mimetype) {
			return mimetype, true
		}
	}

	return mime, false
}

{'fraction_non_alphanumeric': 0.15381145692377085, 'fraction_numerical': 0.0036084799278304014, 'mean_word_length': 3.9730941704035874, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5155196', 'n_tokens_mistral': 919, 'n_tokens_neox': 847, 'n_words': 206}