<commit_before>// Copyright 2015 LinkedIn Corp. Licensed under the Apache License,
// Version 2.0 (the "License"); you may not use this file except in
// compliance with the License.
 You may obtain a copy of the License
// at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// 
distributed under the License is distributed on an "AS IS" BASIS,
// 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache
// License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License.
 You may obtain a copy of
// the License at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// 
distributed under the License is distributed on an "AS IS" BASIS,
// 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.

package goavro

import (
	"bytes"
	"testing"
)

func TestNewReaderBailsBadHeader(t *testing.T) {
	_, err := NewReader(FromReader(new(bytes.Reader)))
	checkError(t, err, "cannot read magic number")

	bits := []byte("BAD\x01\x04\x14avro.codec\x16UNSUPPORTED\x16avro.schema\x0a\x22int\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00")
	_, err = NewReader(FromReader(bytes.NewReader(bits)))
	checkError(t, err, "invalid magic number")

	bits = []byte("Obj\x01\x02")
	_, err = NewReader(FromReader(bytes.NewReader(bits)))
	checkError(t, err, "cannot read header metadata")

	// missing codec
	bits = []byte("Obj\x01\x02\x16avro.schema\x0a\x22int\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00")
	_, err = NewReader(FromReader(bytes.NewReader(bits)))
	checkError(t, err, "header ought to have avro.codec key")

	// unsupported codec
	bits = []byte("Obj\x01\x04\x14avro.codec\x16UNSUPPORTED\x16avro.schema\x0a\x22int\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00")
	_, err = NewReader(FromReader(bytes.NewReader(bits)))
	checkError(t, err, "unsupported codec")

	// missing schema
	bits = []byte("Obj\x01\x02\x14avro.codec\x08null\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00")
	_, err = NewReader(FromReader(bytes.NewReader(bits)))
	checkError(t, err, "header ought to have avro.schema key")

	// schema that doesn't compile
	bits = []byte("Obj\x01\x04\x14avro.codec\x08null\x16avro.schema\x0a\x22???\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00")
	_, err = NewReader(FromReader(bytes.NewReader(bits)))
	checkError(t, err, "cannot compile schema")

	// missing sync marker
	bits = []byte("Obj\x01\x04\x14avro.codec\x08null\x16avro.schema\x0a\x22int\x22\x00")
	_, err = NewReader(FromReader(bytes.NewReader(bits)))
	checkError(t, err, "cannot read sync marker")
}

func TestFileRead(t *testing.T) {
	bits := []byte("\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x05\x7b\x22\x64\x6f\x63\x3a\x22\x3a\x22\x41\x20\x62\x61\x73\x69\x63\x20\x73\x63\x68\x65\x6d\x61\x20\x66\x6f\x72\x20\x73\x74\x6f\x72\x69\x6e\x67\x20\x62\x6c\x6f\x67\x20\x63\x6f\x6d\x6d\x65\x6e\x74\x73\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a\x5b\x7b\x22\x64\x6f\x63\x22\x3a\x22\x4e\x61\x6d\x65\x20\x6f\x66\x20\x75\x73\x65\x72\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x75\x73\x65\x72\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67\x22\x7d\x2c\x7b\x22\x64\x6f\x63\x22\x3a\x22\x54\x68\x65\x20\x63\x6f\x6e\x74\x65\x6e\x74\x20\x6f\x66\x20\x74\x68\x65\x20\x75\x73\x65\x72\x27\x73\x20\x6d\x65\x73\x73\x61\x67\x65\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x63\x6f\x6d\x6d\x65\x6e\x74\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67\x22\x7d\x2c\x7b\x22\x64\x6f\x63\x22\x3a\x22\x55\x6e\x69\x78\x20\x65\x70\x6f\x63\x68\x20\x74\x69\x6d\x65\x20\x69\x6e\x20\x6d\x69\x6c\x6c\x69\x73\x65\x63\x6f\x6e\x64\x73\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x74\x69\x6d\x65\x73\x74\x61\x6d\x70\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x6c\x6f\x6e\x67\x22\x7d\x5d\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x63\x6f\x6d\x6d\x65\x6e\x74\x73\x22\x2c\x22\x6e\x61\x6d\x65\x73\x70\x61\x63\x65\x22\x3a\x22\x63\x6f\x6d\x2e\x65\x78\x61\x6d\x70\x6c\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65\x63\x6f\x72\x64\x22\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x04\xc0\x01\x0e\x41\x71\x75\x61\x6d\x61\x6e\x50\x54\x68\x65\x20\x41\x74\x6c\x61\x6e\x74\x69\x63\x20\x69\x73\x20\x6f\x64\x64\x6c\x79\x20\x63\x6f\x6c\x64\x20\x74\x68\x69\x73\x20\x6d\x6f\x72\x6e\x69\x6e\x67\x21\x88\x88\x88\x88\x08\x0c\x42\x61\x74\x6d\x61\x6e\x3a\x57\x68\x6f\x20\x61\x72\x65\x20\x61\x6c\x6c\x20\x6f\x66\x20\x74\x68\x65\x73\x65\x20\x63\x72\x61\x7a\x69\x65\x73\x3f\x8c\x92\xa1\xd1\x0a\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00")
	bb := bytes.NewBuffer(bits)
	fr, err := NewReader(BufferFromReader(bb))
	if err != nil {
		panic(err)
	}
	defer func() {
		if err := fr.Close(); err != nil {
			panic(err)
		}
	}()
	var count int
	for fr.Scan() {
		datum, err := fr.Read()
		if err != nil {
			t.Errorf("Actual: %#v; Expected: %#v", err, nil)
		}
		_, ok := datum.(*Record)
		if !ok {
			t.Errorf("Actual: %T; Expected: *Record", datum)
		}
		count++
	}
	if count != 2 {
		t.Errorf("Actual: %#v; Expected: %#v", count, 2)
	}
}<commit_msg>test to ensure Reader.Scan never blocks<commit_after>// Copyright 2015 LinkedIn Corp. Licensed under the Apache License,
// Version 2.0 (the "License"); you may not use this file except in
// compliance with the License.
 You may obtain a copy of the License
// at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// 
distributed under the License is distributed on an "AS IS" BASIS,
// 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache
// License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License.
 You may obtain a copy of
// the License at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// 
distributed under the License is distributed on an "AS IS" BASIS,
// 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.

package goavro

import (
	"bytes"
	"testing"
)

func TestNewReaderBailsBadHeader(t *testing.T) {
	_, err := NewReader(FromReader(new(bytes.Reader)))
	checkError(t, err, "cannot read magic number")

	bits := []byte("BAD\x01\x04\x14avro.codec\x16UNSUPPORTED\x16avro.schema\x0a\x22int\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00")
	_, err = NewReader(FromReader(bytes.NewReader(bits)))
	checkError(t, err, "invalid magic number")

	bits = []byte("Obj\x01\x02")
	_, err = NewReader(FromReader(bytes.NewReader(bits)))
	checkError(t, err, "cannot read header metadata")

	// missing codec
	bits = []byte("Obj\x01\x02\x16avro.schema\x0a\x22int\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00")
	_, err = NewReader(FromReader(bytes.NewReader(bits)))
	checkError(t, err, "header ought to have avro.codec key")

	// unsupported codec
	bits = []byte("Obj\x01\x04\x14avro.codec\x16UNSUPPORTED\x16avro.schema\x0a\x22int\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00")
	_, err = NewReader(FromReader(bytes.NewReader(bits)))
	checkError(t, err, "unsupported codec")

	// missing schema
	bits = []byte("Obj\x01\x02\x14avro.codec\x08null\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00")
	_, err = NewReader(FromReader(bytes.NewReader(bits)))
	checkError(t, err, "header ought to have avro.schema key")

	// schema that doesn't compile
	bits = []byte("Obj\x01\x04\x14avro.codec\x08null\x16avro.schema\x0a\x22???\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00")
	_, err = NewReader(FromReader(bytes.NewReader(bits)))
	checkError(t, err, "cannot compile schema")

	// missing sync marker
	bits = []byte("Obj\x01\x04\x14avro.codec\x08null\x16avro.schema\x0a\x22int\x22\x00")
	_, err = NewReader(FromReader(bytes.NewReader(bits)))
	checkError(t, err, "cannot read sync marker")
}

func TestFileRead(t *testing.T) {
	bits := []byte("\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x05\x7b\x22\x64\x6f\x63\x3a\x22\x3a\x22\x41\x20\x62\x61\x73\x69\x63\x20\x73\x63\x68\x65\x6d\x61\x20\x66\x6f\x72\x20\x73\x74\x6f\x72\x69\x6e\x67\x20\x62\x6c\x6f\x67\x20\x63\x6f\x6d\x6d\x65\x6e\x74\x73\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a\x5b\x7b\x22\x64\x6f\x63\x22\x3a\x22\x4e\x61\x6d\x65\x20\x6f\x66\x20\x75\x73\x65\x72\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x75\x73\x65\x72\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67\x22\x7d\x2c\x7b\x22\x64\x6f\x63\x22\x3a\x22\x54\x68\x65\x20\x63\x6f\x6e\x74\x65\x6e\x74\x20\x6f\x66\x20\x74\x68\x65\x20\x75\x73\x65\x72\x27\x73\x20\x6d\x65\x73\x73\x61\x67\x65\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x63\x6f\x6d\x6d\x65\x6e\x74\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67\x22\x7d\x2c\x7b\x22\x64\x6f\x63\x22\x3a\x22\x55\x6e\x69\x78\x20\x65\x70\x6f\x63\x68\x20\x74\x69\x6d\x65\x20\x69\x6e\x20\x6d\x69\x6c\x6c\x69\x73\x65\x63\x6f\x6e\x64\x73\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x74\x69\x6d\x65\x73\x74\x61\x6d\x70\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x6c\x6f\x6e\x67\x22\x7d\x5d\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x63\x6f\x6d\x6d\x65\x6e\x74\x73\x22\x2c\x22\x6e\x61\x6d\x65\x73\x70\x61\x63\x65\x22\x3a\x22\x63\x6f\x6d\x2e\x65\x78\x61\x6d\x70\x6c\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65\x63\x6f\x72\x64\x22\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x04\xc0\x01\x0e\x41\x71\x75\x61\x6d\x61\x6e\x50\x54\x68\x65\x20\x41\x74\x6c\x61\x6e\x74\x69\x63\x20\x69\x73\x20\x6f\x64\x64\x6c\x79\x20\x63\x6f\x6c\x64\x20\x74\x68\x69\x73\x20\x6d\x6f\x72\x6e\x69\x6e\x67\x21\x88\x88\x88\x88\x08\x0c\x42\x61\x74\x6d\x61\x6e\x3a\x57\x68\x6f\x20\x61\x72\x65\x20\x61\x6c\x6c\x20\x6f\x66\x20\x74\x68\x65\x73\x65\x20\x63\x72\x61\x7a\x69\x65\x73\x3f\x8c\x92\xa1\xd1\x0a\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00")
	bb := bytes.NewBuffer(bits)
	fr, err := NewReader(BufferFromReader(bb))
	if err != nil {
		panic(err)
	}
	defer func() {
		if err := fr.Close(); err != nil {
			panic(err)
		}
	}()
	var count int
	for fr.Scan() {
		datum, err := fr.Read()
		if err != nil {
			t.Errorf("Actual: %#v; Expected: %#v", err, nil)
		}
		_, ok := datum.(*Record)
		if !ok {
			t.Errorf("Actual: %T; Expected: *Record", datum)
		}
		count++
	}
	if count != 2 {
		t.Errorf("Actual: %#v; Expected: %#v", count, 2)
	}
}

func TestReaderScanShouldNotBlock(t *testing.T) {
	bits := []byte("Obj\x01\x04\x14avro.codec\x08null\x16avro.schema\x0a\x22int\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00")
	fr, err := NewReader(FromReader(bytes.NewReader(bits)))
	checkErrorFatal(t, err, nil)
	if available := fr.Scan(); available {
		t.Errorf("Actual: %#v; Expected: %#v", available, false)
	}
	if err = fr.Close(); err != nil {
		t.Errorf("Actual: %#v; Expected: %#v", err, nil)
	}
}
{'fraction_non_alphanumeric': 0.19389005602240897, 'fraction_numerical': 0.18907563025210083, 'mean_word_length': 3.5938882187374346, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3697286', 'n_tokens_mistral': 7251, 'n_tokens_neox': 6277, 'n_words': 631}