<commit_before>package sarama

import (
	"bytes"
	"encoding/binary"

	"code.google.com/p/snappy-go/snappy"
)

var snappyMagic = []byte{130, 83, 78, 65, 80, 80, 89, 0}

// SnappyEncode encodes binary data
func snappyEncode(src []byte) ([]byte, error) {
	return snappy.Encode(nil, src)
}

// SnappyDecode decodes snappy data
func snappyDecode(src []byte) ([]byte, error) {
	if bytes.Equal(src[:8], snappyMagic) {
		var (
			pos   = uint32(16)
			max   = uint32(len(src))
			dst   = make([]byte, 0, len(src))
			chunk []byte
			err   error
		)
		for pos < max {
			size := binary.BigEndian.Uint32(src[pos : pos+4])
			pos += 4

			chunk, err = snappy.Decode(chunk, src[pos:pos+size])
			if err != nil {
				return nil, err
			}
			pos += size
			dst = append(dst, chunk...)
		}
		return dst, nil
	}
	return snappy.Decode(nil, src)
}
<commit_msg>fix code.google.com/p/snappy-go/snappy to github.com/golang/snappy<commit_after>package sarama

import (
	"bytes"
	"encoding/binary"

	"github.com/golang/snappy/snappy"
)

var snappyMagic = []byte{130, 83, 78, 65, 80, 80, 89, 0}

// SnappyEncode encodes binary data
func snappyEncode(src []byte) ([]byte, error) {
	return snappy.Encode(nil, src)
}

// SnappyDecode decodes snappy data
func snappyDecode(src []byte) ([]byte, error) {
	if bytes.Equal(src[:8], snappyMagic) {
		var (
			pos   = uint32(16)
			max   = uint32(len(src))
			dst   = make([]byte, 0, len(src))
			chunk []byte
			err   error
		)
		for pos < max {
			size := binary.BigEndian.Uint32(src[pos : pos+4])
			pos += 4

			chunk, err = snappy.Decode(chunk, src[pos:pos+size])
			if err != nil {
				return nil, err
			}
			pos += size
			dst = append(dst, chunk...)
		}
		return dst, nil
	}
	return snappy.Decode(nil, src)
}

{'fraction_non_alphanumeric': 0.15859284890426759, 'fraction_numerical': 0.03229527104959631, 'mean_word_length': 3.2009685230024214, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1289063', 'n_tokens_mistral': 792, 'n_tokens_neox': 668, 'n_words': 167}