<commit_before>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import (
	"bytes"
	"reflect"
	"testing"
	"time"

	"github.com/keybase/client/go/libkb"
	"golang.org/x/net/context"
)

func totalBlockRefs(m map[BlockID]map[BlockRefNonce]blockRefLocalStatus) int {
	n := 0
	for _, refs := range m {
		n += len(refs)
	}
	return n
}

// Test that quota reclamation works for a simple case where the user
// does a few updates, then lets quota reclamation run, and we make
// sure that all historical blocks have been deleted.
func TestQuotaReclamationSimple(t *testing.T) {
	var userName libkb.NormalizedUsername = "test_user"
	config, _, ctx := kbfsOpsInitNoMocks(t, userName)
	defer CheckConfigAndShutdown(t, config)

	clock, now := newTestClockAndTimeNow()
	config.SetClock(clock)

	rootNode := GetRootNodeOrBust(t, config, userName.String(), false)
	kbfsOps := config.KBFSOps()
	_, _, err := kbfsOps.CreateDir(ctx, rootNode, "a")
	if err != nil {
		t.Fatalf("Couldn't create dir: %v", err)
	}
	err = kbfsOps.RemoveDir(ctx, rootNode, "a")
	if err != nil {
		t.Fatalf("Couldn't remove dir: %v", err)
	}

	// Wait for outstanding archives
	err = kbfsOps.SyncFromServerForTesting(ctx, rootNode.GetFolderBranch())
	if err != nil {
		t.Fatalf("Couldn't sync from server: %v", err)
	}

	// Make sure no blocks are deleted before there's a new-enough update.
	bserverLocal, ok := config.BlockServer().(*BlockServerLocal)
	if !ok {
		t.Fatalf("Bad block server")
	}
	preQR1Blocks, err := bserverLocal.getAll(rootNode.GetFolderBranch().Tlf)
	if err != nil {
		t.Fatalf("Couldn't get blocks: %v", err)
	}

	ops := kbfsOps.(*KBFSOpsStandard).getOpsByNode(ctx, rootNode)
	ops.fbm.forceQuotaReclamation()
	err = ops.fbm.waitForQuotaReclamations(ctx)
	if err != nil {
		t.Fatalf("Couldn't wait for QR: %v", err)
	}

	postQR1Blocks, err := bserverLocal.getAll(rootNode.GetFolderBranch().Tlf)
	if err != nil {
		t.Fatalf("Couldn't get blocks: %v", err)
	}

	if !reflect.DeepEqual(preQR1Blocks, postQR1Blocks) {
		t.Fatalf("Blocks deleted too early (%v vs %v)!",
			preQR1Blocks, postQR1Blocks)
	}

	// Increase the time and make a new revision, but don't run quota
	// reclamation yet.
	clock.Set(now.Add(2 * config.QuotaReclamationMinUnrefAge()))
	_, _, err = kbfsOps.CreateDir(ctx, rootNode, "b")
	if err != nil {
		t.Fatalf("Couldn't create dir: %v", err)
	}

	preQR2Blocks, err := bserverLocal.getAll(rootNode.GetFolderBranch().Tlf)
	if err != nil {
		t.Fatalf("Couldn't get blocks: %v", err)
	}

	ops.fbm.forceQuotaReclamation()
	err = ops.fbm.waitForQuotaReclamations(ctx)
	if err != nil {
		t.Fatalf("Couldn't wait for QR: %v", err)
	}

	postQR2Blocks, err := bserverLocal.getAll(rootNode.GetFolderBranch().Tlf)
	if err != nil {
		t.Fatalf("Couldn't get blocks: %v", err)
	}

	if pre, post := totalBlockRefs(preQR2Blocks),
		totalBlockRefs(postQR2Blocks); post >= pre {
		t.Errorf("Blocks didn't shrink after reclamation: pre: %d, post %d",
			pre, post)
	}
}

// Test that a single quota reclamation run doesn't try to reclaim too
// much quota at once.
func TestQuotaReclamationIncrementalReclamation(t *testing.T) {
	var userName libkb.NormalizedUsername = "test_user"
	config, _, ctx := kbfsOpsInitNoMocks(t, userName)
	defer CheckConfigAndShutdown(t, config)

	now := time.Now()
	var clock TestClock
	clock.Set(now)
	config.SetClock(&clock)
<commit_msg>folder_block_manager_test: check for unembedded block change list<commit_after>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import (
	"bytes"
	"reflect"
	"testing"
	"time"

	"github.com/keybase/client/go/libkb"
	"golang.org/x/net/context"
)

func totalBlockRefs(m map[BlockID]map[BlockRefNonce]blockRefLocalStatus) int {
	n := 0
	for _, refs := range m {
		n += len(refs)
	}
	return n
}

// Test that quota reclamation works for a simple case where the user
// does a few updates, then lets quota reclamation run, and we make
// sure that all historical blocks have been deleted.
func testQuotaReclamation(t *testing.T, ctx context.Context, config Config,
	userName libkb.NormalizedUsername) {
	clock, now := newTestClockAndTimeNow()
	config.SetClock(clock)

	rootNode := GetRootNodeOrBust(t, config, userName.String(), false)
	kbfsOps := config.KBFSOps()
	_, _, err := kbfsOps.CreateDir(ctx, rootNode, "a")
	if err != nil {
		t.Fatalf("Couldn't create dir: %v", err)
	}
	err = kbfsOps.RemoveDir(ctx, rootNode, "a")
	if err != nil {
		t.Fatalf("Couldn't remove dir: %v", err)
	}

	// Wait for outstanding archives
	err = kbfsOps.SyncFromServerForTesting(ctx, rootNode.GetFolderBranch())
	if err != nil {
		t.Fatalf("Couldn't sync from server: %v", err)
	}

	// Make sure no blocks are deleted before there's a new-enough update.
	bserverLocal, ok := config.BlockServer().(*BlockServerLocal)
	if !ok {
		t.Fatalf("Bad block server")
	}
	preQR1Blocks, err := bserverLocal.getAll(rootNode.GetFolderBranch().Tlf)
	if err != nil {
		t.Fatalf("Couldn't get blocks: %v", err)
	}

	ops := kbfsOps.(*KBFSOpsStandard).getOpsByNode(ctx, rootNode)
	ops.fbm.forceQuotaReclamation()
	err = ops.fbm.waitForQuotaReclamations(ctx)
	if err != nil {
		t.Fatalf("Couldn't wait for QR: %v", err)
	}

	postQR1Blocks, err := bserverLocal.getAll(rootNode.GetFolderBranch().Tlf)
	if err != nil {
		t.Fatalf("Couldn't get blocks: %v", err)
	}

	if !reflect.DeepEqual(preQR1Blocks, postQR1Blocks) {
		t.Fatalf("Blocks deleted too early (%v vs %v)!",
			preQR1Blocks, postQR1Blocks)
	}

	// Increase the time and make a new revision, but don't run quota
	// reclamation yet.
	clock.Set(now.Add(2 * config.QuotaReclamationMinUnrefAge()))
	_, _, err = kbfsOps.CreateDir(ctx, rootNode, "b")
	if err != nil {
		t.Fatalf("Couldn't create dir: %v", err)
	}

	preQR2Blocks, err := bserverLocal.getAll(rootNode.GetFolderBranch().Tlf)
	if err != nil {
		t.Fatalf("Couldn't get blocks: %v", err)
	}

	ops.fbm.forceQuotaReclamation()
	err = ops.fbm.waitForQuotaReclamations(ctx)
	if err != nil {
		t.Fatalf("Couldn't wait for QR: %v", err)
	}

	postQR2Blocks, err := bserverLocal.getAll(rootNode.GetFolderBranch().Tlf)
	if err != nil {
		t.Fatalf("Couldn't get blocks: %v", err)
	}

	if pre, post := totalBlockRefs(preQR2Blocks),
		totalBlockRefs(postQR2Blocks); post >= pre {
		t.Errorf("Blocks didn't shrink after reclamation: pre: %d, post %d",
			pre, post)
	}
}

func TestQuotaReclamationSimple(t *testing.T) {
	var userName libkb.NormalizedUsername = "test_user"
	config, _, ctx := kbfsOpsInitNoMocks(t, userName)
	defer CheckConfigAndShutdown(t, config)

	testQuotaReclamation(t, ctx, config, userName)
}

// Just like the simple case, except tests that it unembeds large sets
// of pointers correctly.
func TestQuotaReclamationUnembedded(t *testing.T) {
	var userName libkb.NormalizedUsername = "test_user"
	config, _, ctx := kbfsOpsInitNoMocks(t, userName)
	defer CheckConfigAndShutdown(t, config)

	config.(*ConfigLocal).bsplit.(*BlockSplitterSimple).
		blockChangeEmbedMaxSize = 32

	testQuotaReclamation(t, ctx, config, userName)

	// Make sure the MD has an unembedded change block.
	rootNode := GetRootNodeOrBust(t, config, userName.String(), false)
	md, err := config.MDOps().GetForTLF(ctx, rootNode.GetFolderBranch().Tlf)
	if err != nil {
		t.Fatalf("Couldn't get MD: %v", err)
	}
	if md.data.Changes.Info.BlockPointer == zeroPtr {
		t.Fatalf("No unembedded changes for ops %v", md.data.Changes.Ops)
	}
}

// Test that a single quota reclamation run doesn't try to reclaim too
// much quota at once.
func TestQuotaReclamationIncrementalReclamation(t *testing.T) {
	var userName libkb.NormalizedUsername = "test_user"
	config, _, ctx := kbfsOpsInitNoMocks(t, userName)
	defer CheckConfigAndShutdown(t, config)

	now := time.Now()
	var clock TestClock
	clock.Set(now)
	config.SetClock(&clock)

{'fraction_non_alphanumeric': 0.13159879336349925, 'fraction_numerical': 0.004273504273504274, 'mean_word_length': 4.483804272915231, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2254168', 'n_tokens_mistral': 3041, 'n_tokens_neox': 2867, 'n_words': 755}