<commit_before>package ct

// TODO(Martin2112): Tests that verify the signature on SCTs and STHs. All the signing in here
// uses dummy objects. Real tests might be better done as integration tests on the log operation.

import (
	"bufio"
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/golang/glog"
	"github.com/golang/mock/gomock"
	ct "github.com/google/certificate-transparency/go"
	"github.com/google/certificate-transparency/go/fixchain"
	"github.com/google/certificate-transparency/go/tls"
	"github.com/google/certificate-transparency/go/x509"
	"github.com/google/trillian"
	"github.com/google/trillian/crypto"
	"github.com/google/trillian/examples/ct/testonly"
	"github.com/google/trillian/mockclient"
	"github.com/google/trillian/util"
	"golang.org/x/net/context"
)

// Arbitrary time for use in tests
var fakeTime = time.Date(2016, 7, 22, 11, 01, 13, 0, time.UTC)

// The deadline should be the above bumped by 500ms
var fakeDeadlineTime = time.Date(2016, 7, 22, 11, 01, 13, 500*1000*1000, time.UTC)
var fakeTimeSource = util.FakeTimeSource{FakeTime: fakeTime}
var okStatus = &trillian.TrillianApiStatus{StatusCode: trillian.TrillianApiStatusCode_OK}

type getEntriesRangeTestCase struct {
	start          int64
	end            int64
	expectedStatus int
	explanation    string
	rpcExpected    bool
}

var getEntriesRangeTestCases = []getEntriesRangeTestCase{
	{-1, 0, http.StatusBadRequest, "-ve start value not allowed", false},
	{0, -1, http.StatusBadRequest, "-ve end value not allowed", false},
	{20, 10, http.StatusBadRequest, "invalid range end>start", false},
	{3000, -50, http.StatusBadRequest, "invalid range, -ve end", false},
	{10, 20, http.StatusInternalServerError, "valid range", true},
	{10, 10, http.StatusInternalServerError, "valid range, one entry", true},
	{10, 9, http.StatusBadRequest, "invalid range, edge case", false},
	{1000, 50000, http.StatusBadRequest, "range too large to be accepted", false}}

// List of requests for get-entry-and-proof that should be rejected with bad request status
var getEntryAndProofBadRequests = []string{
	"", "leaf_index=b", "leaf_index=1&tree_size=-1", "leaf_index=-1&tree_size=1",
	"leaf_index=1&tree_size=d", "leaf_index=&tree_size=", "leaf_index=", "leaf_index=1&tree_size=0",
	"leaf_index=10&tree_size=5", "leaf_index=tree_size"}

// A list of requests that should result in a bad request status
var getProofByHashBadRequests = []string{"", "hash=&tree_size=1", "hash=''&tree_size=1", "hash=notbase64data&tree_size=1", "tree_size=-1&hash=aGkK"}

// A list of requests for get-sth-consistency that should result in a bad request status
var getSTHConsistencyBadRequests = []string{"", "first=apple&second=orange", "first=1&second=a",
	"first=a&second=2", "first=-1&second=10", "first=10&second=-11", "first=6&second=6",
	"first=998&second=997", "first=1000&second=200", "first=10", "second=20"}

// The result we expect after a roundtrip in the successful get proof by hash test
var expectedInclusionProofByHash = ct.GetProofByHashResponse{
	LeafIndex: 2,
	AuditPath: [][]byte{[]byte("abcdef"), []byte("ghijkl"), []byte("mnopqr")}}

// The result we expect after a roundtrip in the successful get sth consistency test
var expectedSTHConsistencyProofByHash = ct.GetSTHConsistencyResponse{Consistency: [][]byte{[]byte("abcdef"), []byte("ghijkl"), []byte("mnopqr")}}

const caCertB64 string = `<KEY>`

const intermediateCertB64 string = `<KEY>

const caAndIntermediateCertsPEM string = "-----BEGIN CERTIFICATE-----\n" + caCertB64 + "\n-----END CERTIFICATE-----\n" +
	"\n-----BEGIN CERTIFICATE-----\n" + intermediateCertB64 + "\n-----END CERTIFICATE-----\n"

// Used in test of corrupt Merkle leaves
const invalidLeafString string = "NOT A MERKLE TREE LEAF"

type handlerAndPath struct {
	path    string
	handler appHandler
}

func allGetHandlersForTest(trustedRoots *PEMCertPool, c LogContext) []handlerAndPath {
	return []handlerAndPath{
		{"get-sth", appHandler{context: c, handler: getSTH, method: http.MethodGet}},
		{"get-sth-consistency", appHandler{context: c, handler: getSTHConsistency, method: http.MethodGet}},
		{"get-proof-by-hash", appHandler{context: c, handler: getProofByHash, method: http.MethodGet}},
		{"get-entries", appHandler{context: c, handler: getEntries, method: http.MethodGet}},
		{"get-roots", appHandler{context: c, handler: getRoots, method: http.MethodGet}},
		{"get-entry-and-proof", appHandler{context: c, handler: getEntryAndProof, method: http.MethodGet}},
	}
}

func allPostHandlersForTest(client trillian.TrillianLogClient) []handlerAndPath {
	pool := NewPEMCertPool()
	ok := pool.AppendCertsFromPEM([]byte(testonly.FakeCACertPEM))
	if !ok {
		glog.Fatal("Failed to load cert pool")
	}
	c := LogContext{rpcClient: client, trustedRoots: pool, timeSource: fakeTimeSource}

	return []handlerAndPath{
		{"add-chain", appHandler{context: c, handler: addChain, method: http.MethodPost}},
		{"add-pre-chain", appHandler{context: c, handler: addPreChain, method: http.MethodPost}},
	}
}

func TestPostHandlersOnlyAcceptPost(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	client := mockclient.NewMockTrillianLogClient(mockCtrl)

	// Anything in the post handler list should only accept POST
	for _, hp := range allPostHandlersForTest(client) {
		s := httptest.NewServer(hp.handler)
		defer s.Close()

		resp, err := http.Get(s.URL + "/ct/v1/" + hp.path)
		if err != nil {
			t.Fatal(err)
		}

		// TODO(Martin2112): Remove this test when there are no more handlers to be implemented and
		// rely on the handlers own tests
		if expected, got := http.StatusMethodNotAllowed, resp.StatusCode; expected != got {
			t.Fatalf("Wrong status code for GET to POST handler, expected %v got %v", expected, got)
		}

		resp, err = http.Post(s.URL+"/ct/v1/"+hp.path, "application/json", nil)
		if err != nil {
			t.Fatal(err)
		}

		if expected, got := http.StatusBadRequest, resp.StatusCode; expected != got {
			t.Fatalf("Wrong status code for POST to POST handler, expected %v got %v", expected, got)
		}
	}
}

func TestGetHandlersRejectPost(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	client := mockclient.NewMockTrillianLogClient(mockCtrl)
	pool := NewPEMCertPool()
	handlers := LogContext{rpcClient: client, timeSource: fakeTimeSource}

	// Anything in the get handler list should not accept POST. We don't test they accept
	// GET because that needs different mock backend set up per handler.
	for _, hp := range allGetHandlersForTest(pool, handlers) {
		s := httptest.NewServer(hp.handler)
		defer s.Close()
<commit_msg>examples/ct: fix misleading variable name<commit_after>package ct

// TODO(Martin2112): Tests that verify the signature on SCTs and STHs. All the signing in here
// uses dummy objects. Real tests might be better done as integration tests on the log operation.

import (
	"bufio"
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/golang/glog"
	"github.com/golang/mock/gomock"
	ct "github.com/google/certificate-transparency/go"
	"github.com/google/certificate-transparency/go/fixchain"
	"github.com/google/certificate-transparency/go/tls"
	"github.com/google/certificate-transparency/go/x509"
	"github.com/google/trillian"
	"github.com/google/trillian/crypto"
	"github.com/google/trillian/examples/ct/testonly"
	"github.com/google/trillian/mockclient"
	"github.com/google/trillian/util"
	"golang.org/x/net/context"
)

// Arbitrary time for use in tests
var fakeTime = time.Date(2016, 7, 22, 11, 01, 13, 0, time.UTC)

// The deadline should be the above bumped by 500ms
var fakeDeadlineTime = time.Date(2016, 7, 22, 11, 01, 13, 500*1000*1000, time.UTC)
var fakeTimeSource = util.FakeTimeSource{FakeTime: fakeTime}
var okStatus = &trillian.TrillianApiStatus{StatusCode: trillian.TrillianApiStatusCode_OK}

type getEntriesRangeTestCase struct {
	start          int64
	end            int64
	expectedStatus int
	explanation    string
	rpcExpected    bool
}

var getEntriesRangeTestCases = []getEntriesRangeTestCase{
	{-1, 0, http.StatusBadRequest, "-ve start value not allowed", false},
	{0, -1, http.StatusBadRequest, "-ve end value not allowed", false},
	{20, 10, http.StatusBadRequest, "invalid range end>start", false},
	{3000, -50, http.StatusBadRequest, "invalid range, -ve end", false},
	{10, 20, http.StatusInternalServerError, "valid range", true},
	{10, 10, http.StatusInternalServerError, "valid range, one entry", true},
	{10, 9, http.StatusBadRequest, "invalid range, edge case", false},
	{1000, 50000, http.StatusBadRequest, "range too large to be accepted", false}}

// List of requests for get-entry-and-proof that should be rejected with bad request status
var getEntryAndProofBadRequests = []string{
	"", "leaf_index=b", "leaf_index=1&tree_size=-1", "leaf_index=-1&tree_size=1",
	"leaf_index=1&tree_size=d", "leaf_index=&tree_size=", "leaf_index=", "leaf_index=1&tree_size=0",
	"leaf_index=10&tree_size=5", "leaf_index=tree_size"}

// A list of requests that should result in a bad request status
var getProofByHashBadRequests = []string{"", "hash=&tree_size=1", "hash=''&tree_size=1", "hash=notbase64data&tree_size=1", "tree_size=-1&hash=aGkK"}

// A list of requests for get-sth-consistency that should result in a bad request status
var getSTHConsistencyBadRequests = []string{"", "first=apple&second=orange", "first=1&second=a",
	"first=a&second=2", "first=-1&second=10", "first=10&second=-11", "first=6&second=6",
	"first=998&second=997", "first=1000&second=200", "first=10", "second=20"}

// The result we expect after a roundtrip in the successful get proof by hash test
var expectedInclusionProofByHash = ct.GetProofByHashResponse{
	LeafIndex: 2,
	AuditPath: [][]byte{[]byte("abcdef"), []byte("ghijkl"), []byte("mnopqr")}}

// The result we expect after a roundtrip in the successful get sth consistency test
var expectedSTHConsistencyProofByHash = ct.GetSTHConsistencyResponse{Consistency: [][]byte{[]byte("abcdef"), []byte("ghijkl"), []byte("mnopqr")}}

const caCertB64 string = `MIIC0DCCAjmgAwIBAgIBADANBgkqhkiG9w0BAQUFADBVMQswCQYDVQQGEwJHQjEk
MCIGA1UEChMbQ2VydGlmaWNhdGUgVHJhbnNwYXJlbmN5IENBMQ4wDAYDVQQIEwVX
YWxlczEQMA4GA1UEBxMHRXJ3IFdlbjAeFw0xMjA2MDEwMDAwMDBaFw0yMjA2MDEw
MDAwMDBaMFUxCzAJBgNVBAYTAkdCMSQwIgYDVQQKExtDZXJ0aWZpY2F0ZSBUcmFu
c3BhcmVuY3kgQ0ExDjAMBgNVBAgTBVdhbGVzMRAwDgYDVQQHEwdFcncgV2VuMIGf
MA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDVimhTYhCicRmTbneDIRgcKkATxtB7
jHbrkVfT0PtLO1FuzsvRyY2RxS90P6tjXVUJnNE6uvMa5UFEJFGnTHgW8iQ8+EjP
KDHM5nugSlojgZ88ujfmJNnDvbKZuDnd/iYx0ss6hPx7srXFL8/BT/9Ab1zURmnL
<KEY>`

const intermediateCertB64 string = `<KEY>

const caAndIntermediateCertsPEM string = "-----BEGIN CERTIFICATE-----\n" + caCertB64 + "\n-----END CERTIFICATE-----\n" +
	"\n-----BEGIN CERTIFICATE-----\n" + intermediateCertB64 + "\n-----END CERTIFICATE-----\n"

// Used in test of corrupt Merkle leaves
const invalidLeafString string = "NOT A MERKLE TREE LEAF"

type handlerAndPath struct {
	path    string
	handler appHandler
}

func allGetHandlersForTest(trustedRoots *PEMCertPool, c LogContext) []handlerAndPath {
	return []handlerAndPath{
		{"get-sth", appHandler{context: c, handler: getSTH, method: http.MethodGet}},
		{"get-sth-consistency", appHandler{context: c, handler: getSTHConsistency, method: http.MethodGet}},
		{"get-proof-by-hash", appHandler{context: c, handler: getProofByHash, method: http.MethodGet}},
		{"get-entries", appHandler{context: c, handler: getEntries, method: http.MethodGet}},
		{"get-roots", appHandler{context: c, handler: getRoots, method: http.MethodGet}},
		{"get-entry-and-proof", appHandler{context: c, handler: getEntryAndProof, method: http.MethodGet}},
	}
}

func allPostHandlersForTest(client trillian.TrillianLogClient) []handlerAndPath {
	pool := NewPEMCertPool()
	ok := pool.AppendCertsFromPEM([]byte(testonly.FakeCACertPEM))
	if !ok {
		glog.Fatal("Failed to load cert pool")
	}
	c := LogContext{rpcClient: client, trustedRoots: pool, timeSource: fakeTimeSource}

	return []handlerAndPath{
		{"add-chain", appHandler{context: c, handler: addChain, method: http.MethodPost}},
		{"add-pre-chain", appHandler{context: c, handler: addPreChain, method: http.MethodPost}},
	}
}

func TestPostHandlersOnlyAcceptPost(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	client := mockclient.NewMockTrillianLogClient(mockCtrl)

	// Anything in the post handler list should only accept POST
	for _, hp := range allPostHandlersForTest(client) {
		s := httptest.NewServer(hp.handler)
		defer s.Close()

		resp, err := http.Get(s.URL + "/ct/v1/" + hp.path)
		if err != nil {
			t.Fatal(err)
		}

		// TODO(Martin2112): Remove this test when there are no more handlers to be implemented and
		// rely on the handlers own tests
		if expected, got := http.StatusMethodNotAllowed, resp.StatusCode; expected != got {
			t.Fatalf("Wrong status code for GET to POST handler, expected %v got %v", expected, got)
		}

		resp, err = http.Post(s.URL+"/ct/v1/"+hp.path, "application/json", nil)
		if err != nil {
			t.Fatal(err)
		}

		if expected, got := http.StatusBadRequest, resp.StatusCode; expected != got {
			t.Fatalf("Wrong status code for POST to POST handler, expected %v got %v", expected, got)
		}
	}
}

func TestGetHandlersRejectPost(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	client := mockclient.NewMockTrillianLogClient(mockCtrl)
	pool := NewPEMCertPool()
	c := LogContext{rpcClient: client, timeSource: fakeTimeSource}

	// Anything in the get handler list should not accept POST. We don't test they accept
	// GET because that needs different mock backend set up per handler.
	for _, hp := range allGetHandlersForTest(pool, c) {
		s := httptest.NewServer(hp.handler)
		defer s.Close()

{'fraction_non_alphanumeric': 0.13586800573888091, 'fraction_numerical': 0.024390243902439025, 'mean_word_length': 5.2487673688928735, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2951574', 'n_tokens_mistral': 5294, 'n_tokens_neox': 4884, 'n_words': 1190}