<commit_before>package torrentfs

import (
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"testing"
	"time"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	_ "github.com/anacrolix/envpprof"
	"github.com/anacrolix/missinggo"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	netContext "golang.org/x/net/context"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/internal/testutil"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/storage"
)

func init() {
	log.SetFlags(log.Flags() | log.Lshortfile)
}

func TestTCPAddrString(t *testing.T) {
	l, err := net.Listen("tcp4", "localhost:0")
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()
	c, err := net.Dial("tcp", l.Addr().String())
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()
	ras := c.RemoteAddr().String()
	ta := &net.TCPAddr{
		IP:   net.IPv4(127, 0, 0, 1),
		Port: missinggo.AddrPort(l.Addr()),
	}
	s := ta.String()
	if ras != s {
		t.FailNow()
	}
}

type testLayout struct {
	BaseDir   string
	MountDir  string
	Completed string
	Metainfo  *metainfo.MetaInfo
}

func (tl *testLayout) Destroy() error {
	return os.RemoveAll(tl.BaseDir)
}

func newGreetingLayout() (tl testLayout, err error) {
	tl.BaseDir, err = ioutil.TempDir("", "torrentfs")
	if err != nil {
		return
	}
	tl.Completed = filepath.Join(tl.BaseDir, "completed")
	os.Mkdir(tl.Completed, 0777)
	tl.MountDir = filepath.Join(tl.BaseDir, "mnt")
	os.Mkdir(tl.MountDir, 0777)
	testutil.CreateDummyTorrentData(tl.Completed)
	tl.Metainfo = testutil.GreetingMetaInfo()
	return
}

// Unmount without first killing the FUSE connection while there are FUSE
// operations blocked inside the filesystem code.
func TestUnmountWedged(t *testing.T) {
	layout, err := newGreetingLayout()
	require.NoError(t, err)
	defer func() {
		err := layout.Destroy()
		if err != nil {
			t.Log(err)
		}
	}()
	client, err := torrent.NewClient(&torrent.Config{
		DataDir:         filepath.Join(layout.BaseDir, "incomplete"),
		DisableTrackers: true,
		NoDHT:           true,
		DisableTCP:      true,
		DisableUTP:      true,
	})
	require.NoError(t, err)
	defer client.Close()
	tt, err := client.AddTorrent(layout.Metainfo)
	require.NoError(t, err)
	fs := New(client)
	fuseConn, err := fuse.Mount(layout.MountDir)
	if err != nil {
		switch err.Error() {
		case "cannot locate OSXFUSE",
			"fusermount: exit status 1":
			t.Skip(err)
		}
		t.Fatal(err)
	}
	go func() {
		server := fusefs.New(fuseConn, &fusefs.Config{
			Debug: func(msg interface{}) {
				t.Log(msg)
			},
		})
		server.Serve(fs)
	}()
	<-fuseConn.Ready
	if err := fuseConn.MountError; err != nil {
		t.Fatalf("mount error: %s", err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	// Read the greeting file, though it will never be available. This should
	// "wedge" FUSE, requiring the fs object to be forcibly destroyed. The
	// read call will return with a FS error.
	go func() {
		<-ctx.Done()
		fs.mu.Lock()
		fs.event.Broadcast()
		fs.mu.Unlock()
	}()
	go func() {
		defer cancel()
		_, err := ioutil.ReadFile(filepath.Join(layout.MountDir, tt.Info().Name))
		if err == nil {
			t.Fatal("expected error reading greeting")
		}
	}()

	// Wait until the read has blocked inside the filesystem code.
	fs.mu.Lock()
	for fs.blockedReads != 1 && ctx.Err() == nil {
		fs.event.Wait()
	}
	fs.mu.Unlock()

	fs.Destroy()

	for {
		err = fuse.Unmount(layout.MountDir)
		if err != nil {
			t.Logf("error unmounting: %s", err)
			time.Sleep(time.Millisecond)
		} else {
			break
		}
	}

	err = fuseConn.Close()
	if err != nil {
		t.Fatalf("error closing fuse conn: %s", err)
	}
}

func TestDownloadOnDemand(t *testing.T) {
	layout, err := newGreetingLayout()
	require.NoError(t, err)
	defer layout.Destroy()
	seeder, err := torrent.NewClient(&torrent.Config{
		DataDir:         layout.Completed,
		DisableTrackers: true,
		NoDHT:           true,
		ListenAddr:      "localhost:0",
		Seed:            true,
	})
	require.NoError(t, err)
	defer seeder.Close()
	testutil.ExportStatusWriter(seeder, "s")
	_, err = seeder.AddMagnet(fmt.Sprintf("magnet:?xt=urn:btih:%s", layout.Metainfo.HashInfoBytes().HexString()))
	require.NoError(t, err)
	leecher, err := torrent.NewClient(&torrent.Config{
		DisableTrackers: true,
		NoDHT:           true,
		ListenAddr:      "localhost:0",
		DisableTCP:      true,
		DefaultStorage:  storage.NewMMap(filepath.Join(layout.BaseDir, "download")),
		// This can be used to check if clients can connect to other clients
		// with the same ID.
		// PeerID: seeder.PeerID(),
	})
	require.NoError(t, err)
	testutil.ExportStatusWriter(leecher, "l")
	defer leecher.Close()
	leecherTorrent, _ := leecher.AddTorrent(layout.Metainfo)
	leecherTorrent.AddPeers([]torrent.Peer{
		{
			IP:   missinggo.AddrIP(seeder.ListenAddr()),
			Port: missinggo.AddrPort(seeder.ListenAddr()),
		},
	})
	fs := New(leecher)
	defer fs.Destroy()
	root, _ := fs.Root()
	node, _ := root.(fusefs.NodeStringLookuper).Lookup(netContext.Background(), "greeting")
	var attr fuse.Attr
	node.Attr(netContext.Background(), &attr)
	size := attr.Size
	resp := &fuse.ReadResponse{
		Data: make([]byte, size),
	}
	h, err := node.(fusefs.NodeOpener).Open(nil, nil, nil)
	require.NoError(t, err)
	h.(fusefs.HandleReader).Read(netContext.Background(), &fuse.ReadRequest{
		Size: int(size),
	}, resp)
	assert.EqualValues(t, testutil.GreetingFileContents, resp.Data)
}

func TestIsSubPath(t *testing.T) {
	for _, case_ := range []struct {
		parent, child string
		is            bool
	}{
		{"", "", false},
		{"", "/", true},
		{"", "a", true},
		{"a/b", "a/bc", false},
		{"a/b", "a/b", false},
		{"a/b", "a/b/c", true},
		{"a/b", "a//b", false},
	} {
		assert.Equal(t, case_.is, isSubPath(case_.parent, case_.child))
	}
}
<commit_msg>fs: Fix tests now that data verification isn't automatic<commit_after>package torrentfs

import (
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"testing"
	"time"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	_ "github.com/anacrolix/envpprof"
	"github.com/anacrolix/missinggo"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	netContext "golang.org/x/net/context"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/internal/testutil"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/storage"
)

func init() {
	log.SetFlags(log.Flags() | log.Lshortfile)
}

func TestTCPAddrString(t *testing.T) {
	l, err := net.Listen("tcp4", "localhost:0")
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()
	c, err := net.Dial("tcp", l.Addr().String())
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()
	ras := c.RemoteAddr().String()
	ta := &net.TCPAddr{
		IP:   net.IPv4(127, 0, 0, 1),
		Port: missinggo.AddrPort(l.Addr()),
	}
	s := ta.String()
	if ras != s {
		t.FailNow()
	}
}

type testLayout struct {
	BaseDir   string
	MountDir  string
	Completed string
	Metainfo  *metainfo.MetaInfo
}

func (tl *testLayout) Destroy() error {
	return os.RemoveAll(tl.BaseDir)
}

func newGreetingLayout() (tl testLayout, err error) {
	tl.BaseDir, err = ioutil.TempDir("", "torrentfs")
	if err != nil {
		return
	}
	tl.Completed = filepath.Join(tl.BaseDir, "completed")
	os.Mkdir(tl.Completed, 0777)
	tl.MountDir = filepath.Join(tl.BaseDir, "mnt")
	os.Mkdir(tl.MountDir, 0777)
	testutil.CreateDummyTorrentData(tl.Completed)
	tl.Metainfo = testutil.GreetingMetaInfo()
	return
}

// Unmount without first killing the FUSE connection while there are FUSE
// operations blocked inside the filesystem code.
func TestUnmountWedged(t *testing.T) {
	layout, err := newGreetingLayout()
	require.NoError(t, err)
	defer func() {
		err := layout.Destroy()
		if err != nil {
			t.Log(err)
		}
	}()
	client, err := torrent.NewClient(&torrent.Config{
		DataDir:         filepath.Join(layout.BaseDir, "incomplete"),
		DisableTrackers: true,
		NoDHT:           true,
		DisableTCP:      true,
		DisableUTP:      true,
	})
	require.NoError(t, err)
	defer client.Close()
	tt, err := client.AddTorrent(layout.Metainfo)
	require.NoError(t, err)
	fs := New(client)
	fuseConn, err := fuse.Mount(layout.MountDir)
	if err != nil {
		switch err.Error() {
		case "cannot locate OSXFUSE",
			"fusermount: exit status 1":
			t.Skip(err)
		}
		t.Fatal(err)
	}
	go func() {
		server := fusefs.New(fuseConn, &fusefs.Config{
			Debug: func(msg interface{}) {
				t.Log(msg)
			},
		})
		server.Serve(fs)
	}()
	<-fuseConn.Ready
	if err := fuseConn.MountError; err != nil {
		t.Fatalf("mount error: %s", err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	// Read the greeting file, though it will never be available. This should
	// "wedge" FUSE, requiring the fs object to be forcibly destroyed. The
	// read call will return with a FS error.
	go func() {
		<-ctx.Done()
		fs.mu.Lock()
		fs.event.Broadcast()
		fs.mu.Unlock()
	}()
	go func() {
		defer cancel()
		_, err := ioutil.ReadFile(filepath.Join(layout.MountDir, tt.Info().Name))
		if err == nil {
			t.Fatal("expected error reading greeting")
		}
	}()

	// Wait until the read has blocked inside the filesystem code.
	fs.mu.Lock()
	for fs.blockedReads != 1 && ctx.Err() == nil {
		fs.event.Wait()
	}
	fs.mu.Unlock()

	fs.Destroy()

	for {
		err = fuse.Unmount(layout.MountDir)
		if err != nil {
			t.Logf("error unmounting: %s", err)
			time.Sleep(time.Millisecond)
		} else {
			break
		}
	}

	err = fuseConn.Close()
	if err != nil {
		t.Fatalf("error closing fuse conn: %s", err)
	}
}

func TestDownloadOnDemand(t *testing.T) {
	layout, err := newGreetingLayout()
	require.NoError(t, err)
	defer layout.Destroy()
	seeder, err := torrent.NewClient(&torrent.Config{
		DataDir:         layout.Completed,
		DisableTrackers: true,
		NoDHT:           true,
		ListenAddr:      "localhost:0",
		Seed:            true,
	})
	require.NoError(t, err)
	defer seeder.Close()
	testutil.ExportStatusWriter(seeder, "s")
	// Just to mix things up, the seeder starts with the data, but the leecher
	// starts with the metainfo.
	seederTorrent, err := seeder.AddMagnet(fmt.Sprintf("magnet:?xt=urn:btih:%s", layout.Metainfo.HashInfoBytes().HexString()))
	require.NoError(t, err)
	go func() {
		// Wait until we get the metainfo, then check for the data.
		<-seederTorrent.GotInfo()
		seederTorrent.VerifyData()
	}()
	leecher, err := torrent.NewClient(&torrent.Config{
		DisableTrackers: true,
		NoDHT:           true,
		ListenAddr:      "localhost:0",
		DisableTCP:      true,
		DefaultStorage:  storage.NewMMap(filepath.Join(layout.BaseDir, "download")),
		// This can be used to check if clients can connect to other clients
		// with the same ID.
		// PeerID: seeder.PeerID(),
	})
	require.NoError(t, err)
	testutil.ExportStatusWriter(leecher, "l")
	defer leecher.Close()
	leecherTorrent, _ := leecher.AddTorrent(layout.Metainfo)
	leecherTorrent.AddPeers([]torrent.Peer{
		{
			IP:   missinggo.AddrIP(seeder.ListenAddr()),
			Port: missinggo.AddrPort(seeder.ListenAddr()),
		},
	})
	fs := New(leecher)
	defer fs.Destroy()
	root, _ := fs.Root()
	node, _ := root.(fusefs.NodeStringLookuper).Lookup(netContext.Background(), "greeting")
	var attr fuse.Attr
	node.Attr(netContext.Background(), &attr)
	size := attr.Size
	resp := &fuse.ReadResponse{
		Data: make([]byte, size),
	}
	h, err := node.(fusefs.NodeOpener).Open(nil, nil, nil)
	require.NoError(t, err)
	h.(fusefs.HandleReader).Read(netContext.Background(), &fuse.ReadRequest{
		Size: int(size),
	}, resp)
	assert.EqualValues(t, testutil.GreetingFileContents, resp.Data)
}

func TestIsSubPath(t *testing.T) {
	for _, case_ := range []struct {
		parent, child string
		is            bool
	}{
		{"", "", false},
		{"", "/", true},
		{"", "a", true},
		{"a/b", "a/bc", false},
		{"a/b", "a/b", false},
		{"a/b", "a/b/c", true},
		{"a/b", "a//b", false},
	} {
		assert.Equal(t, case_.is, isSubPath(case_.parent, case_.child))
	}
}

{'fraction_non_alphanumeric': 0.1638736842105263, 'fraction_numerical': 0.003536842105263158, 'mean_word_length': 3.6719118804091266, 'pattern_counts': {'":': 2, '<': 8, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2316787', 'n_tokens_mistral': 4856, 'n_tokens_neox': 4590, 'n_words': 958}