<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A collection of tests for a file system where we do not attempt to write to
// the file system at all. Rather we set up contents in a GCS bucket out of
// band, wait for them to be available, and then read them via the file system.
//
// These tests are registered by RegisterFSTests.

package fstesting

import (
	"encoding/hex"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"strings"
	"syscall"

	"github.com/jacobsa/gcloud/gcs/gcsutil"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type foreignModsTest struct {
	fsTest
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *foreignModsTest) StatRoot() {
	fi, err := os.Stat(t.mfs.Dir())
	AssertEq(nil, err)

	ExpectEq(path.Base(t.mfs.Dir()), fi.Name())
	ExpectEq(0, fi.Size())
	ExpectEq(0700|os.ModeDir, fi.Mode())
	ExpectTrue(fi.IsDir())
	ExpectEq(1, fi.Sys().(*syscall.Stat_t).Nlink)
	ExpectEq(currentUid(), fi.Sys().(*syscall.Stat_t).Uid)
	ExpectEq(currentGid(), fi.Sys().(*syscall.Stat_t).Gid)
}

func (t *foreignModsTest) ReadDir_EmptyRoot() {
	// ReadDir
	entries, err := t.readDirUntil(0, t.mfs.Dir())
	AssertEq(nil, err)

	ExpectThat(entries, ElementsAre())
}

func (t *foreignModsTest) ReadDir_ContentsInRoot() {
	// Set up contents.
	createTime := t.clock.Now()
	AssertEq(
		nil,
		t.createObjects(
			map[string]string{
				// File
				"foo": "taco",

				// Directory
				"bar/": "",

				// File
				"baz": "burrito",
			}))

	// Make sure the time below doesn't match.
	t.advanceTime()

	/////////////////////////
	// ReadDir
	/////////////////////////

	entries, err := t.readDirUntil(3, t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(3, len(entries), "Names: %v", getFileNames(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(0700|os.ModeDir, e.Mode())
	ExpectTrue(e.IsDir())
	ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
	ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
	ExpectEq(currentGid(), e.Sys().(*syscall.Stat_t).Gid)

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0700), e.Mode())
	ExpectThat(e.ModTime(), t.matchesStartTime(createTime))
	ExpectFalse(e.IsDir())
	ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
	ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
	ExpectEq(currentGid(), e.Sys().(*syscall.Stat_t).Gid)

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0700), e.Mode())
	ExpectThat(e.ModTime(), t.matchesStartTime(createTime))
	ExpectFalse(e.IsDir())
	ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
	ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
	ExpectEq(currentGid(), e.Sys().(*syscall.Stat_t).Gid)
}

func (t *foreignModsTest) ReadDir_EmptySubDirectory() {
	// Set up an empty directory placeholder called 'bar'.
	AssertEq(nil, t.createEmptyObjects([]string{"bar/"}))

	// ReadDir
	entries, err := t.readDirUntil(1, t.mfs.Dir())
	AssertEq(nil, err)
	AssertEq(1, len(entries))

	entries, err = t.readDirUntil(0, path.Join(t.mfs.Dir(), "bar"))
	AssertEq(nil, err)

	ExpectThat(entries, ElementsAre())
}

func (t *foreignModsTest) ReadDir_ContentsInSubDirectory() {
	// Set up contents.
	createTime := t.clock.Now()
	AssertEq(
		nil,
		t.createObjects(
			map[string]string{
				// Placeholder
				"dir/": "",

				// File
				"dir/foo": "taco",

				// Directory
				"dir/bar/": "",

				// File
				"dir/baz": "burrito",
			}))

	// Make sure the time below doesn't match.
	t.advanceTime()

	// Wait for the directory to show up in the file system.
	_, err := t.readDirUntil(1, path.Join(t.mfs.Dir()))
	AssertEq(nil, err)

	// ReadDir
	entries, err := t.readDirUntil(3, path.Join(t.mfs.Dir(), "dir"))
	AssertEq(nil, err)

	AssertEq(3, len(entries), "Names: %v", getFileNames(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(0700|os.ModeDir, e.Mode())
	ExpectTrue(e.IsDir())
	ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
	ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
	ExpectEq(currentGid(), e.Sys().(*syscall.Stat_t).Gid)

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0700), e.Mode())
	ExpectThat(e.ModTime(), t.matchesStartTime(createTime))
	ExpectFalse(e.IsDir())
	ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
	ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
	ExpectEq(currentGid(), e.Sys().(*syscall.Stat_t).Gid)

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0700), e.Mode())
	ExpectThat(e.ModTime(), t.matchesStartTime(createTime))
	ExpectFalse(e.IsDir())
	ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
	ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
	ExpectEq(currentGid(), e.Sys().(*syscall.Stat_t).Gid)
}

func (t *foreignModsTest) UnreachableObjects() {
	// Set up objects that appear to be directory contents, but for which there
	// is no directory placeholder object. We don't have implicit directories
	// enabled, so these should be unreachable.
	err := gcsutil.CreateEmptyObjects(
		t.ctx,
		t.bucket,
		[]string{
			"foo/0",
			"foo/1",
			"bar/0/",
		})

	AssertEq(nil, err)

	// Nothing should show up in the root.
	_, err = t.readDirUntil(0, path.Join(t.mfs.Dir()))
	AssertEq(nil, err)

	// Statting the directories shouldn't work.
	_, err = os.Stat(path.Join(t.mfs.Dir(), "foo"))

	AssertNe(nil, err)
	ExpectTrue(os.IsNotExist(err), "err: %v", err)
}

func (t *foreignModsTest) FileAndDirectoryWithConflictingName() {
	var fi os.FileInfo
	var entries []os.FileInfo
	var err error

	// Set up an object named "foo" and one named "foo/", plus a child for the
	// latter.
	AssertEq(
		nil,
		t.createObjects(
			map[string]string{
				// File
				"foo": "taco",

				// Directory
				"foo/": "",

				// Directory child
				"foo/bar": "",
			}))

	// A listing of the parent should contain a directory named "foo" and a
	// file named "foo\n".
	entries, err = ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)
	AssertEq(2, len(entries))

	fi = entries[0]
	ExpectEq("foo", fi.Name())
	ExpectEq(0, fi.Size())
	ExpectEq(0700|os.ModeDir, fi.Mode())
	ExpectTrue(fi.IsDir())
	ExpectEq(1, fi.Sys().(*syscall.Stat_t).Nlink)

	fi = entries[1]
	ExpectEq("foo\n", fi.Name())
	ExpectEq(len("taco"), fi.Size())
	ExpectEq(len("taco"), fi.Size())
	ExpectEq(os.FileMode(0700), fi.Mode())
	ExpectFalse(fi.IsDir())
	ExpectEq(1, fi.Sys().(*syscall.Stat_t).Nlink)

	// Statting "foo" should yield the directory.
	fi, err = os.Stat(path.Join(t.mfs.Dir(), "foo"))
	AssertEq(nil, err)

	ExpectEq("foo", fi.Name())
	ExpectTrue(fi.IsDir())

	// Statting "foo\n" should yield the file.
	fi, err = os.Stat(path.Join(t.mfs.Dir(), "foo\n"))
	AssertEq(nil, err)

	ExpectEq("foo\n", fi.Name())
	ExpectEq(len("taco"), fi.Size())
	ExpectFalse(fi.IsDir())
}

func (t *foreignModsTest) StatTrailingNewlineName_NoConflictingNames() {
	var err error

	// Set up an object named "foo".
	AssertEq(
		nil,
		t.createObjects(
			map[string]string{
				"foo": "taco",
			}))

	// We shouldn't be able to stat "foo\n", because there is no conflicting
	// directory name.
	_, err = os.Stat(path.Join(t.mfs.Dir(), "foo\n"))
	ExpectTrue(os.IsNotExist(err), "err: %v", err)
}
<commit_msg>Expanded a test case for #30. Doesn't reproduce the issue.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A collection of tests for a file system where we do not attempt to write to
// the file system at all. Rather we set up contents in a GCS bucket out of
// band, wait for them to be available, and then read them via the file system.
//
// These tests are registered by RegisterFSTests.

package fstesting

import (
	"encoding/hex"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"strings"
	"syscall"

	"github.com/jacobsa/gcloud/gcs/gcsutil"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type foreignModsTest struct {
	fsTest
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *foreignModsTest) StatRoot() {
	fi, err := os.Stat(t.mfs.Dir())
	AssertEq(nil, err)

	ExpectEq(path.Base(t.mfs.Dir()), fi.Name())
	ExpectEq(0, fi.Size())
	ExpectEq(0700|os.ModeDir, fi.Mode())
	ExpectTrue(fi.IsDir())
	ExpectEq(1, fi.Sys().(*syscall.Stat_t).Nlink)
	ExpectEq(currentUid(), fi.Sys().(*syscall.Stat_t).Uid)
	ExpectEq(currentGid(), fi.Sys().(*syscall.Stat_t).Gid)
}

func (t *foreignModsTest) ReadDir_EmptyRoot() {
	// ReadDir
	entries, err := t.readDirUntil(0, t.mfs.Dir())
	AssertEq(nil, err)

	ExpectThat(entries, ElementsAre())
}

func (t *foreignModsTest) ReadDir_ContentsInRoot() {
	// Set up contents.
	createTime := t.clock.Now()
	AssertEq(
		nil,
		t.createObjects(
			map[string]string{
				// File
				"foo": "taco",

				// Directory
				"bar/": "",

				// File
				"baz": "burrito",
			}))

	// Make sure the time below doesn't match.
	t.advanceTime()

	/////////////////////////
	// ReadDir
	/////////////////////////

	entries, err := t.readDirUntil(3, t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(3, len(entries), "Names: %v", getFileNames(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(0700|os.ModeDir, e.Mode())
	ExpectTrue(e.IsDir())
	ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
	ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
	ExpectEq(currentGid(), e.Sys().(*syscall.Stat_t).Gid)

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0700), e.Mode())
	ExpectThat(e.ModTime(), t.matchesStartTime(createTime))
	ExpectFalse(e.IsDir())
	ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
	ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
	ExpectEq(currentGid(), e.Sys().(*syscall.Stat_t).Gid)

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0700), e.Mode())
	ExpectThat(e.ModTime(), t.matchesStartTime(createTime))
	ExpectFalse(e.IsDir())
	ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
	ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
	ExpectEq(currentGid(), e.Sys().(*syscall.Stat_t).Gid)
}

func (t *foreignModsTest) ReadDir_EmptySubDirectory() {
	// Set up an empty directory placeholder called 'bar'.
	AssertEq(nil, t.createEmptyObjects([]string{"bar/"}))

	// ReadDir
	entries, err := t.readDirUntil(1, t.mfs.Dir())
	AssertEq(nil, err)
	AssertEq(1, len(entries))

	entries, err = t.readDirUntil(0, path.Join(t.mfs.Dir(), "bar"))
	AssertEq(nil, err)

	ExpectThat(entries, ElementsAre())
}

func (t *foreignModsTest) ReadDir_ContentsInSubDirectory() {
	// Set up contents.
	createTime := t.clock.Now()
	AssertEq(
		nil,
		t.createObjects(
			map[string]string{
				// Placeholder
				"dir/": "",

				// File
				"dir/foo": "taco",

				// Directory
				"dir/bar/": "",

				// File
				"dir/baz": "burrito",
			}))

	// Make sure the time below doesn't match.
	t.advanceTime()

	// Wait for the directory to show up in the file system.
	_, err := t.readDirUntil(1, path.Join(t.mfs.Dir()))
	AssertEq(nil, err)

	// ReadDir
	entries, err := t.readDirUntil(3, path.Join(t.mfs.Dir(), "dir"))
	AssertEq(nil, err)

	AssertEq(3, len(entries), "Names: %v", getFileNames(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(0700|os.ModeDir, e.Mode())
	ExpectTrue(e.IsDir())
	ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
	ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
	ExpectEq(currentGid(), e.Sys().(*syscall.Stat_t).Gid)

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0700), e.Mode())
	ExpectThat(e.ModTime(), t.matchesStartTime(createTime))
	ExpectFalse(e.IsDir())
	ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
	ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
	ExpectEq(currentGid(), e.Sys().(*syscall.Stat_t).Gid)

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0700), e.Mode())
	ExpectThat(e.ModTime(), t.matchesStartTime(createTime))
	ExpectFalse(e.IsDir())
	ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
	ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
	ExpectEq(currentGid(), e.Sys().(*syscall.Stat_t).Gid)
}

func (t *foreignModsTest) UnreachableObjects() {
	// Set up objects that appear to be directory contents, but for which there
	// is no directory placeholder object. We don't have implicit directories
	// enabled, so these should be unreachable.
	err := gcsutil.CreateEmptyObjects(
		t.ctx,
		t.bucket,
		[]string{
			"foo/0",
			"foo/1",
			"bar/0/",
		})

	AssertEq(nil, err)

	// Nothing should show up in the root.
	_, err = t.readDirUntil(0, path.Join(t.mfs.Dir()))
	AssertEq(nil, err)

	// Statting the directories shouldn't work.
	_, err = os.Stat(path.Join(t.mfs.Dir(), "foo"))

	AssertNe(nil, err)
	ExpectTrue(os.IsNotExist(err), "err: %v", err)
}

func (t *foreignModsTest) FileAndDirectoryWithConflictingName() {
	var fi os.FileInfo
	var entries []os.FileInfo
	var err error

	// Set up an object named "foo" and one named "foo/", plus a child for the
	// latter.
	AssertEq(
		nil,
		t.createObjects(
			map[string]string{
				// File
				"foo": "taco",

				// Directory
				"foo/": "",

				// Directory child
				"foo/bar": "burrito",
			}))

	// A listing of the parent should contain a directory named "foo" and a
	// file named "foo\n".
	entries, err = ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)
	AssertEq(2, len(entries))

	fi = entries[0]
	ExpectEq("foo", fi.Name())
	ExpectEq(0, fi.Size())
	ExpectEq(0700|os.ModeDir, fi.Mode())
	ExpectTrue(fi.IsDir())
	ExpectEq(1, fi.Sys().(*syscall.Stat_t).Nlink)

	fi = entries[1]
	ExpectEq("foo\n", fi.Name())
	ExpectEq(len("taco"), fi.Size())
	ExpectEq(os.FileMode(0700), fi.Mode())
	ExpectFalse(fi.IsDir())
	ExpectEq(1, fi.Sys().(*syscall.Stat_t).Nlink)

	// Statting "foo" should yield the directory.
	fi, err = os.Stat(path.Join(t.mfs.Dir(), "foo"))
	AssertEq(nil, err)

	ExpectEq("foo", fi.Name())
	ExpectTrue(fi.IsDir())

	// Statting "foo\n" should yield the file.
	fi, err = os.Stat(path.Join(t.mfs.Dir(), "foo\n"))
	AssertEq(nil, err)

	ExpectEq("foo\n", fi.Name())
	ExpectEq(len("taco"), fi.Size())
	ExpectFalse(fi.IsDir())

	// Listing the directory should yield the sole child file.
	entries, err = ioutil.ReadDir(path.Join(t.Dir, "foo"))
	AssertEq(nil, err)
	AssertEq(1, len(entries))

	fi = entries[0]
	ExpectEq("bar", fi.Name())
	ExpectEq(len("burrito"), fi.Size())
	ExpectEq(os.FileMode(0700), fi.Mode())
	ExpectFalse(fi.IsDir())
	ExpectEq(1, fi.Sys().(*syscall.Stat_t).Nlink)
}

func (t *foreignModsTest) StatTrailingNewlineName_NoConflictingNames() {
	var err error

	// Set up an object named "foo".
	AssertEq(
		nil,
		t.createObjects(
			map[string]string{
				"foo": "taco",
			}))

	// We shouldn't be able to stat "foo\n", because there is no conflicting
	// directory name.
	_, err = os.Stat(path.Join(t.mfs.Dir(), "foo\n"))
	ExpectTrue(os.IsNotExist(err), "err: %v", err)
}

{'fraction_non_alphanumeric': 0.21628726773017865, 'fraction_numerical': 0.009977893290314871, 'mean_word_length': 3.3094747682801238, 'pattern_counts': {'":': 22, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '233401', 'n_tokens_mistral': 6593, 'n_tokens_neox': 6103, 'n_words': 1366}