<commit_before>// Copyright 2019 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package submission

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"regexp"
	"testing"
	"time"

	"github.com/golang/glog"
	"github.com/google/certificate-transparency-go/client"
	"github.com/google/certificate-transparency-go/ctpolicy"
	"github.com/google/certificate-transparency-go/loglist3"
	"github.com/google/certificate-transparency-go/schedule"
	"github.com/google/certificate-transparency-go/testdata"
	"github.com/google/certificate-transparency-go/x509"
	"github.com/google/certificate-transparency-go/x509util"
	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
	"github.com/google/trillian/monitoring"
)

func newLocalStubLogClient(log *loglist3.Log) (client.AddLogClient, error) {
	return newRootedStubLogClient(log, RootsCerts)
}

func ExampleDistributor() {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	d, err := NewDistributor(sampleValidLogList(), buildStubCTPolicy(1), newLocalStubLogClient, monitoring.InertMetricFactory{})
	if err != nil {
		panic(err)
	}

	// Refresh roots periodically so they stay up-to-date.
	// Not necessary for this example, but appropriate for long-running systems.
	refresh := make(chan struct{})
	go schedule.Every(ctx, time.Hour, func(ctx context.Context) {
		if errs := d.RefreshRoots(ctx); len(errs) > 0 {
			glog.Error(errs)
		}
		refresh <- struct{}{}
	})

	select {
	case <-refresh:
		break
	case <-ctx.Done():
		panic("Context expired")
	}

	scts, err := d.AddPreChain(ctx, pemFileToDERChain("../trillian/testdata/subleaf-pre.chain"), false /* loadPendingLogs */)
	if err != nil {
		panic(err)
	}
	for _, sct := range scts {
		fmt.Printf("%s\n", *sct)
	}
	// Output:
	// {https://ct.googleapis.com/rocketeer/ {Version:0 LogId:aHR0cHM6Ly9jdC5nb29nbGVhcGlzLmNvbS9yb2NrZXQ= Timestamp:1234 Extensions:'' Signature:{{SHA256 ECDSA} []}}}
}

var (
	RootsCerts = map[string][]rootInfo{
		"https://ct.googleapis.com/aviator/": {
			rootInfo{filename: "../trillian/testdata/fake-ca-1.cert"},
			rootInfo{filename: "testdata/some.cert"},
		},
		"https://ct.googleapis.com/rocketeer/": {
			rootInfo{filename: "../trillian/testdata/fake-ca.cert"},
			rootInfo{filename: "../trillian/testdata/fake-ca-1.cert"},
			rootInfo{filename: "testdata/some.cert"},
			rootInfo{filename: "testdata/another.cert"},
		},
		"https://ct.googleapis.com/icarus/": {
			rootInfo{raw: []byte("invalid000")},
			rootInfo{filename: "testdata/another.cert"},
		},
		"uncollectable-roots/log/": {
			rootInfo{raw: []byte("invalid")},
		},
	}
)

// newNoLogClient is LogClientBuilder that always fails.
func newNoLogClient(_ *loglist3.Log) (client.AddLogClient, error) {
	return nil, errors.New("bad log-client builder")
}

func sampleLogList() *loglist3.LogList {
	var ll loglist3.LogList
	if err := json.Unmarshal([]byte(testdata.SampleLogList2), &ll); err != nil {
		panic(fmt.Errorf("unable to Unmarshal testdata.SampleLogList: %v", err))
	}
	return &ll
}<commit_msg>Update to testdata.SampleLogList3 in distributor_test.go (#966)<commit_after>// Copyright 2019 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package submission

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"regexp"
	"testing"
	"time"

	"github.com/golang/glog"
	"github.com/google/certificate-transparency-go/client"
	"github.com/google/certificate-transparency-go/ctpolicy"
	"github.com/google/certificate-transparency-go/loglist3"
	"github.com/google/certificate-transparency-go/schedule"
	"github.com/google/certificate-transparency-go/testdata"
	"github.com/google/certificate-transparency-go/x509"
	"github.com/google/certificate-transparency-go/x509util"
	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
	"github.com/google/trillian/monitoring"
)

func newLocalStubLogClient(log *loglist3.Log) (client.AddLogClient, error) {
	return newRootedStubLogClient(log, RootsCerts)
}

func ExampleDistributor() {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	d, err := NewDistributor(sampleValidLogList(), buildStubCTPolicy(1), newLocalStubLogClient, monitoring.InertMetricFactory{})
	if err != nil {
		panic(err)
	}

	// Refresh roots periodically so they stay up-to-date.
	// Not necessary for this example, but appropriate for long-running systems.
	refresh := make(chan struct{})
	go schedule.Every(ctx, time.Hour, func(ctx context.Context) {
		if errs := d.RefreshRoots(ctx); len(errs) > 0 {
			glog.Error(errs)
		}
		refresh <- struct{}{}
	})

	select {
	case <-refresh:
		break
	case <-ctx.Done():
		panic("Context expired")
	}

	scts, err := d.AddPreChain(ctx, pemFileToDERChain("../trillian/testdata/subleaf-pre.chain"), false /* loadPendingLogs */)
	if err != nil {
		panic(err)
	}
	for _, sct := range scts {
		fmt.Printf("%s\n", *sct)
	}
	// Output:
	// {https://ct.googleapis.com/rocketeer/ {Version:0 LogId:aHR0cHM6Ly9jdC5nb29nbGVhcGlzLmNvbS9yb2NrZXQ= Timestamp:1234 Extensions:'' Signature:{{SHA256 ECDSA} []}}}
}

var (
	RootsCerts = map[string][]rootInfo{
		"https://ct.googleapis.com/aviator/": {
			rootInfo{filename: "../trillian/testdata/fake-ca-1.cert"},
			rootInfo{filename: "testdata/some.cert"},
		},
		"https://ct.googleapis.com/rocketeer/": {
			rootInfo{filename: "../trillian/testdata/fake-ca.cert"},
			rootInfo{filename: "../trillian/testdata/fake-ca-1.cert"},
			rootInfo{filename: "testdata/some.cert"},
			rootInfo{filename: "testdata/another.cert"},
		},
		"https://ct.googleapis.com/icarus/": {
			rootInfo{raw: []byte("invalid000")},
			rootInfo{filename: "testdata/another.cert"},
		},
		"uncollectable-roots/log/": {
			rootInfo{raw: []byte("invalid")},
		},
	}
)

// newNoLogClient is LogClientBuilder that always fails.
func newNoLogClient(_ *loglist3.Log) (client.AddLogClient, error) {
	return nil, errors.New("bad log-client builder")
}

func sampleLogList() *loglist3.LogList {
	var ll loglist3.LogList
	if err := json.Unmarshal([]byte(testdata.SampleLogList3), &ll); err != nil {
		panic(fmt.Errorf("unable to Unmarshal testdata.SampleLogList3: %v", err))
	}
	return &ll
}
{'fraction_non_alphanumeric': 0.15186883423648742, 'fraction_numerical': 0.012644157287758789, 'mean_word_length': 4.387724550898204, 'pattern_counts': {'":': 8, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2111464', 'n_tokens_mistral': 2649, 'n_tokens_neox': 2477, 'n_words': 521}