<commit_before>package main

import (
	"context"
	"flag"
	"log"

	"github.com/google/trillian"
	"google.golang.org/grpc"
)

var (
	trillianLog = flag.String("trillian_log", "localhost:8090", "address of the Trillian Log RPC server.")
	logID       = flag.Int64("log_id", 0, "Trillian LogID to populate.")
)

const CHUNK = 10

func main() {
	flag.Parse()

	g, err := grpc.Dial(*trillianLog, grpc.WithInsecure())
	if err != nil {
		log.Fatalf("Failed to dial Trillian Log: %v", err)
	}
	defer g.Close()

	tc := trillian.NewTrillianLogClient(g)
	ctx := context.Background()

	rr := &trillian.GetLatestSignedLogRootRequest{LogId: *logID}
	lr, err := tc.GetLatestSignedLogRoot(ctx, rr)
	if err != nil {
		log.Fatalf("Can't get log root: %v", err)
	}

	ts := lr.SignedLogRoot.TreeSize
	for n := int64(0); n < ts; {
		g := &trillian.GetLeavesByRangeRequest{LogId: *logID, StartIndex: n, Count: CHUNK}
		r, err := tc.GetLeavesByRange(ctx, g)
		if err != nil {
			log.Fatalf("Can't get leaf %d: %v", n, err)
		}

		// deal with server skew
		if r.Skew.GetTreeSizeSet() {
			ts = r.Skew.GetTreeSize()
			log.Printf("Skew")
		}

		if n < ts && len(r.Leaves) == 0 {
			log.Fatalf("No progress at leaf %d", n)
		}

		for m := 0; m < len(r.Leaves) && n < ts; n++ {
			if r.Leaves[m] == nil {
				log.Fatalf("Can't get leaf %d (no error)", n)
			}
			log.Printf("leaf %d: %v", n, r.Leaves[m])
			m++
		}
	}
}<commit_msg>Update skew handling to new API.<commit_after>package main

import (
	"context"
	"flag"
	"log"

	"github.com/google/trillian"
	"google.golang.org/grpc"
)

var (
	trillianLog = flag.String("trillian_log", "localhost:8090", "address of the Trillian Log RPC server.")
	logID       = flag.Int64("log_id", 0, "Trillian LogID to populate.")
)

const CHUNK = 10

func main() {
	flag.Parse()

	g, err := grpc.Dial(*trillianLog, grpc.WithInsecure())
	if err != nil {
		log.Fatalf("Failed to dial Trillian Log: %v", err)
	}
	defer g.Close()

	tc := trillian.NewTrillianLogClient(g)
	ctx := context.Background()

	rr := &trillian.GetLatestSignedLogRootRequest{LogId: *logID}
	lr, err := tc.GetLatestSignedLogRoot(ctx, rr)
	if err != nil {
		log.Fatalf("Can't get log root: %v", err)
	}

	ts := lr.SignedLogRoot.TreeSize
	for n := int64(0); n < ts; {
		g := &trillian.GetLeavesByRangeRequest{LogId: *logID, StartIndex: n, Count: CHUNK}
		r, err := tc.GetLeavesByRange(ctx, g)
		if err != nil {
			log.Fatalf("Can't get leaf %d: %v", n, err)
		}

		// deal with server skew
		if r.SignedLogRoot.TreeSize < ts {
			ts = r.SignedLogRoot.TreeSize
			log.Printf("Skew")
		}

		if n < ts && len(r.Leaves) == 0 {
			log.Fatalf("No progress at leaf %d", n)
		}

		for m := 0; m < len(r.Leaves) && n < ts; n++ {
			if r.Leaves[m] == nil {
				log.Fatalf("Can't get leaf %d (no error)", n)
			}
			log.Printf("leaf %d: %v", n, r.Leaves[m])
			m++
		}
	}
}
{'fraction_non_alphanumeric': 0.17483256961579133, 'fraction_numerical': 0.00986958054282693, 'mean_word_length': 3.3863987635239567, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5492845', 'n_tokens_mistral': 1288, 'n_tokens_neox': 1175, 'n_words': 286}