testing/fstest: new package for testing file system code

This change adds basic test helpers for file system code.

The type MapFS is a simple map-based file system for use when
exercising general file system code.

The func TestFS tests a file system implementation.

For #41190.

Change-Id: I5a2036f57e733915ad508651ad7317749794423c
Reviewed-on: https://go-review.googlesource.com/c/go/+/243910
Trust: Russ Cox <rsc@golang.org>
Reviewed-by: Rob Pike <r@golang.org>

src/go/build/deps_test.go

@@ -468,7 +468,8 @@ var depsRules = `
 
 	# Test-only
 	log
-	< testing/iotest;
+	< testing/iotest
+	< testing/fstest;
 
 	FMT, flag, math/rand
 	< testing/quick;

src/testing/fstest/mapfs.go

@@ -0,0 +1,204 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fstest
+
+import (
+	"io"
+	"io/fs"
+	"path"
+	"sort"
+	"strings"
+	"time"
+)
+
+// A MapFS is a simple in-memory file system for use in tests,
+// represented as a map from path names (arguments to Open)
+// to information about the files or directories they represent.
+//
+// The map need not include parent directories for files contained
+// in the map; those will be synthesized if needed.
+// But a directory can still be included by setting the MapFile.Mode's ModeDir bit;
+// this may be necessary for detailed control over the directory's FileInfo
+// or to create an empty directory.
+//
+// File system operations read directly from the map,
+// so that the file system can be changed by editing the map as needed.
+// An implication is that file system operations must not run concurrently
+// with changes to the map, which would be a race.
+// Another implication is that opening or reading a directory requires
+// iterating over the entire map, so a MapFS should typically be used with not more
+// than a few hundred entries or directory reads.
+type MapFS map[string]*MapFile
+
+// A MapFile describes a single file in a MapFS.
+type MapFile struct {
+	Data    []byte      // file content
+	Mode    fs.FileMode // FileInfo.Mode
+	ModTime time.Time   // FileInfo.ModTime
+	Sys     interface{} // FileInfo.Sys
+}
+
+var _ fs.FS = MapFS(nil)
+var _ fs.File = (*openMapFile)(nil)
+
+// Open opens the named file.
+func (fsys MapFS) Open(name string) (fs.File, error) {
+	if !fs.ValidPath(name) {
+		return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
+	}
+	file := fsys[name]
+	if file != nil && file.Mode&fs.ModeDir == 0 {
+		// Ordinary file
+		return &openMapFile{name, mapFileInfo{path.Base(name), file}, 0}, nil
+	}
+
+	// Directory, possibly synthesized.
+	// Note that file can be nil here: the map need not contain explicit parent directories for all its files.
+	// But file can also be non-nil, in case the user wants to set metadata for the directory explicitly.
+	// Either way, we need to construct the list of children of this directory.
+	var list []mapFileInfo
+	var elem string
+	var need = make(map[string]bool)
+	if name == "." {
+		elem = "."
+		for fname, f := range fsys {
+			i := strings.Index(fname, "/")
+			if i < 0 {
+				list = append(list, mapFileInfo{fname, f})
+			} else {
+				need[fname[:i]] = true
+			}
+		}
+	} else {
+		elem = name[strings.LastIndex(name, "/")+1:]
+		prefix := name + "/"
+		for fname, f := range fsys {
+			if strings.HasPrefix(fname, prefix) {
+				felem := fname[len(prefix):]
+				i := strings.Index(felem, "/")
+				if i < 0 {
+					list = append(list, mapFileInfo{felem, f})
+				} else {
+					need[fname[len(prefix):len(prefix)+i]] = true
+				}
+			}
+		}
+		// If the directory name is not in the map,
+		// and there are no children of the name in the map,
+		// then the directory is treated as not existing.
+		if file == nil && list == nil && len(need) == 0 {
+			return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
+		}
+	}
+	for _, fi := range list {
+		delete(need, fi.name)
+	}
+	for name := range need {
+		list = append(list, mapFileInfo{name, &MapFile{Mode: fs.ModeDir}})
+	}
+	sort.Slice(list, func(i, j int) bool {
+		return list[i].name < list[j].name
+	})
+
+	if file == nil {
+		file = &MapFile{Mode: fs.ModeDir}
+	}
+	return &mapDir{name, mapFileInfo{elem, file}, list, 0}, nil
+}
+
+// A mapFileInfo implements fs.FileInfo and fs.DirEntry for a given map file.
+type mapFileInfo struct {
+	name string
+	f    *MapFile
+}
+
+func (i *mapFileInfo) Name() string               { return i.name }
+func (i *mapFileInfo) Size() int64                { return int64(len(i.f.Data)) }
+func (i *mapFileInfo) Mode() fs.FileMode          { return i.f.Mode }
+func (i *mapFileInfo) Type() fs.FileMode          { return i.f.Mode.Type() }
+func (i *mapFileInfo) ModTime() time.Time         { return i.f.ModTime }
+func (i *mapFileInfo) IsDir() bool                { return i.f.Mode&fs.ModeDir != 0 }
+func (i *mapFileInfo) Sys() interface{}           { return i.f.Sys }
+func (i *mapFileInfo) Info() (fs.FileInfo, error) { return i, nil }
+
+// An openMapFile is a regular (non-directory) fs.File open for reading.
+type openMapFile struct {
+	path string
+	mapFileInfo
+	offset int64
+}
+
+func (f *openMapFile) Stat() (fs.FileInfo, error) { return &f.mapFileInfo, nil }
+
+func (f *openMapFile) Close() error { return nil }
+
+func (f *openMapFile) Read(b []byte) (int, error) {
+	if f.offset >= int64(len(f.f.Data)) {
+		return 0, io.EOF
+	}
+	if f.offset < 0 {
+		return 0, &fs.PathError{Op: "read", Path: f.path, Err: fs.ErrInvalid}
+	}
+	n := copy(b, f.f.Data[f.offset:])
+	f.offset += int64(n)
+	return n, nil
+}
+
+func (f *openMapFile) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	case 0:
+		// offset += 0
+	case 1:
+		offset += f.offset
+	case 2:
+		offset += int64(len(f.f.Data))
+	}
+	if offset < 0 || offset > int64(len(f.f.Data)) {
+		return 0, &fs.PathError{Op: "seek", Path: f.path, Err: fs.ErrInvalid}
+	}
+	f.offset = offset
+	return offset, nil
+}
+
+func (f *openMapFile) ReadAt(b []byte, offset int64) (int, error) {
+	if offset < 0 || offset > int64(len(f.f.Data)) {
+		return 0, &fs.PathError{Op: "read", Path: f.path, Err: fs.ErrInvalid}
+	}
+	n := copy(b, f.f.Data[offset:])
+	if n < len(b) {
+		return n, io.EOF
+	}
+	return n, nil
+}
+
+// A mapDir is a directory fs.File (so also an fs.ReadDirFile) open for reading.
+type mapDir struct {
+	path string
+	mapFileInfo
+	entry  []mapFileInfo
+	offset int
+}
+
+func (d *mapDir) Stat() (fs.FileInfo, error) { return &d.mapFileInfo, nil }
+func (d *mapDir) Close() error               { return nil }
+func (d *mapDir) Read(b []byte) (int, error) {
+	return 0, &fs.PathError{Op: "read", Path: d.path, Err: fs.ErrInvalid}
+}
+
+func (d *mapDir) ReadDir(count int) ([]fs.DirEntry, error) {
+	n := len(d.entry) - d.offset
+	if count > 0 && n > count {
+		n = count
+	}
+	if n == 0 && count > 0 {
+		return nil, io.EOF
+	}
+	list := make([]fs.DirEntry, n)
+	for i := range list {
+		list[i] = &d.entry[d.offset+i]
+	}
+	d.offset += n
+	return list, nil
+}

src/testing/fstest/mapfs_test.go

@@ -0,0 +1,19 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fstest
+
+import (
+	"testing"
+)
+
+func TestMapFS(t *testing.T) {
+	m := MapFS{
+		"hello":             {Data: []byte("hello, world\n")},
+		"fortune/k/ken.txt": {Data: []byte("If a program is too slow, it must have a loop.\n")},
+	}
+	if err := TestFS(m, "hello", "fortune/k/ken.txt"); err != nil {
+		t.Fatal(err)
+	}
+}

src/testing/fstest/testfs.go

@@ -0,0 +1,364 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package fstest implements support for testing implementations and users of file systems.
+package fstest
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"io/ioutil"
+	"sort"
+	"strings"
+	"testing/iotest"
+)
+
+// TestFS tests a file system implementation.
+// It walks the entire tree of files in fsys,
+// opening and checking that each file behaves correctly.
+// It also checks that the file system contains at least the expected files.
+//
+// If TestFS finds any misbehaviors, it returns an error reporting all of them.
+// The error text spans multiple lines, one per detected misbehavior.
+//
+// Typical usage inside a test is:
+//
+//	if err := fstest.TestFS(myFS, "file/that/should/be/present"); err != nil {
+//		t.Fatal(err)
+//	}
+//
+func TestFS(fsys fs.FS, expected ...string) error {
+	t := fsTester{fsys: fsys}
+	t.checkDir(".")
+	t.checkOpen(".")
+	found := make(map[string]bool)
+	for _, dir := range t.dirs {
+		found[dir] = true
+	}
+	for _, file := range t.files {
+		found[file] = true
+	}
+	for _, name := range expected {
+		if !found[name] {
+			t.errorf("expected but not found: %s", name)
+		}
+	}
+	if len(t.errText) == 0 {
+		return nil
+	}
+	return errors.New("TestFS found errors:\n" + string(t.errText))
+}
+
+// An fsTester holds state for running the test.
+type fsTester struct {
+	fsys    fs.FS
+	errText []byte
+	dirs    []string
+	files   []string
+}
+
+// errorf adds an error line to errText.
+func (t *fsTester) errorf(format string, args ...interface{}) {
+	if len(t.errText) > 0 {
+		t.errText = append(t.errText, '\n')
+	}
+	t.errText = append(t.errText, fmt.Sprintf(format, args...)...)
+}
+
+func (t *fsTester) openDir(dir string) fs.ReadDirFile {
+	f, err := t.fsys.Open(dir)
+	if err != nil {
+		t.errorf("%s: Open: %v", dir, err)
+		return nil
+	}
+	d, ok := f.(fs.ReadDirFile)
+	if !ok {
+		f.Close()
+		t.errorf("%s: Open returned File type %T, not a io.ReadDirFile", dir, f)
+		return nil
+	}
+	return d
+}
+
+// checkDir checks the directory dir, which is expected to exist
+// (it is either the root or was found in a directory listing with IsDir true).
+func (t *fsTester) checkDir(dir string) {
+	// Read entire directory.
+	t.dirs = append(t.dirs, dir)
+	d := t.openDir(dir)
+	if d == nil {
+		return
+	}
+	list, err := d.ReadDir(-1)
+	if err != nil {
+		d.Close()
+		t.errorf("%s: ReadDir(-1): %v", dir, err)
+		return
+	}
+
+	// Check all children.
+	var prefix string
+	if dir == "." {
+		prefix = ""
+	} else {
+		prefix = dir + "/"
+	}
+	for _, info := range list {
+		name := info.Name()
+		switch {
+		case name == ".", name == "..", name == "":
+			t.errorf("%s: ReadDir: child has invalid name: %#q", dir, name)
+			continue
+		case strings.Contains(name, "/"):
+			t.errorf("%s: ReadDir: child name contains slash: %#q", dir, name)
+			continue
+		case strings.Contains(name, `\`):
+			t.errorf("%s: ReadDir: child name contains backslash: %#q", dir, name)
+			continue
+		}
+		path := prefix + name
+		t.checkStat(path, info)
+		t.checkOpen(path)
+		if info.IsDir() {
+			t.checkDir(path)
+		} else {
+			t.checkFile(path)
+		}
+	}
+
+	// Check ReadDir(-1) at EOF.
+	list2, err := d.ReadDir(-1)
+	if len(list2) > 0 || err != nil {
+		d.Close()
+		t.errorf("%s: ReadDir(-1) at EOF = %d entries, %v, wanted 0 entries, nil", dir, len(list2), err)
+		return
+	}
+
+	// Check ReadDir(1) at EOF (different results).
+	list2, err = d.ReadDir(1)
+	if len(list2) > 0 || err != io.EOF {
+		d.Close()
+		t.errorf("%s: ReadDir(1) at EOF = %d entries, %v, wanted 0 entries, EOF", dir, len(list2), err)
+		return
+	}
+
+	// Check that close does not report an error.
+	if err := d.Close(); err != nil {
+		t.errorf("%s: Close: %v", dir, err)
+	}
+
+	// Check that closing twice doesn't crash.
+	// The return value doesn't matter.
+	d.Close()
+
+	// Reopen directory, read a second time, make sure contents match.
+	if d = t.openDir(dir); d == nil {
+		return
+	}
+	defer d.Close()
+	list2, err = d.ReadDir(-1)
+	if err != nil {
+		t.errorf("%s: second Open+ReadDir(-1): %v", dir, err)
+		return
+	}
+	t.checkDirList(dir, "first Open+ReadDir(-1) vs second Open+ReadDir(-1)", list, list2)
+
+	// Reopen directory, read a third time in pieces, make sure contents match.
+	if d = t.openDir(dir); d == nil {
+		return
+	}
+	defer d.Close()
+	list2 = nil
+	for {
+		n := 1
+		if len(list2) > 0 {
+			n = 2
+		}
+		frag, err := d.ReadDir(n)
+		if len(frag) > n {
+			t.errorf("%s: third Open: ReadDir(%d) after %d: %d entries (too many)", dir, n, len(list2), len(frag))
+			return
+		}
+		list2 = append(list2, frag...)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.errorf("%s: third Open: ReadDir(%d) after %d: %v", dir, n, len(list2), err)
+			return
+		}
+		if n == 0 {
+			t.errorf("%s: third Open: ReadDir(%d) after %d: 0 entries but nil error", dir, n, len(list2))
+			return
+		}
+	}
+	t.checkDirList(dir, "first Open+ReadDir(-1) vs third Open+ReadDir(1,2) loop", list, list2)
+}
+
+// formatEntry formats an fs.DirEntry into a string for error messages and comparison.
+func formatEntry(entry fs.DirEntry) string {
+	return fmt.Sprintf("%s IsDir=%v Type=%v", entry.Name(), entry.IsDir(), entry.Type())
+}
+
+// formatInfoEntry formats an fs.FileInfo into a string like the result of formatEntry, for error messages and comparison.
+func formatInfoEntry(info fs.FileInfo) string {
+	return fmt.Sprintf("%s IsDir=%v Type=%v", info.Name(), info.IsDir(), info.Mode().Type())
+}
+
+// formatInfo formats an fs.FileInfo into a string for error messages and comparison.
+func formatInfo(info fs.FileInfo) string {
+	return fmt.Sprintf("%s IsDir=%v Mode=%v Size=%d ModTime=%v", info.Name(), info.IsDir(), info.Mode(), info.Size(), info.ModTime())
+}
+
+// checkStat checks that a direct stat of path matches entry,
+// which was found in the parent's directory listing.
+func (t *fsTester) checkStat(path string, entry fs.DirEntry) {
+	file, err := t.fsys.Open(path)
+	if err != nil {
+		t.errorf("%s: Open: %v", path, err)
+		return
+	}
+	info, err := file.Stat()
+	file.Close()
+	if err != nil {
+		t.errorf("%s: Stat: %v", path, err)
+		return
+	}
+	fentry := formatEntry(entry)
+	finfo := formatInfoEntry(info)
+	if fentry != finfo {
+		t.errorf("%s: mismatch:\n\tentry = %v\n\tfile.Stat() = %v", path, fentry, finfo)
+	}
+}
+
+// checkDirList checks that two directory lists contain the same files and file info.
+// The order of the lists need not match.
+func (t *fsTester) checkDirList(dir, desc string, list1, list2 []fs.DirEntry) {
+	old := make(map[string]fs.DirEntry)
+	checkMode := func(entry fs.DirEntry) {
+		if entry.IsDir() != (entry.Type()&fs.ModeDir != 0) {
+			if entry.IsDir() {
+				t.errorf("%s: ReadDir returned %s with IsDir() = true, Type() & ModeDir = 0", dir, entry.Name())
+			} else {
+				t.errorf("%s: ReadDir returned %s with IsDir() = false, Type() & ModeDir = ModeDir", dir, entry.Name())
+			}
+		}
+	}
+
+	for _, entry1 := range list1 {
+		old[entry1.Name()] = entry1
+		checkMode(entry1)
+	}
+
+	var diffs []string
+	for _, entry2 := range list2 {
+		entry1 := old[entry2.Name()]
+		if entry1 == nil {
+			checkMode(entry2)
+			diffs = append(diffs, "+ "+formatEntry(entry2))
+			continue
+		}
+		if formatEntry(entry1) != formatEntry(entry2) {
+			diffs = append(diffs, "- "+formatEntry(entry1), "+ "+formatEntry(entry2))
+		}
+		delete(old, entry2.Name())
+	}
+	for _, entry1 := range old {
+		diffs = append(diffs, "- "+formatEntry(entry1))
+	}
+
+	if len(diffs) == 0 {
+		return
+	}
+
+	sort.Slice(diffs, func(i, j int) bool {
+		fi := strings.Fields(diffs[i])
+		fj := strings.Fields(diffs[j])
+		// sort by name (i < j) and then +/- (j < i, because + < -)
+		return fi[1]+" "+fj[0] < fj[1]+" "+fi[0]
+	})
+
+	t.errorf("%s: diff %s:\n\t%s", dir, desc, strings.Join(diffs, "\n\t"))
+}
+
+// checkFile checks that basic file reading works correctly.
+func (t *fsTester) checkFile(file string) {
+	t.files = append(t.files, file)
+
+	// Read entire file.
+	f, err := t.fsys.Open(file)
+	if err != nil {
+		t.errorf("%s: Open: %v", file, err)
+		return
+	}
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		f.Close()
+		t.errorf("%s: Open+ReadAll: %v", file, err)
+		return
+	}
+
+	if err := f.Close(); err != nil {
+		t.errorf("%s: Close: %v", file, err)
+	}
+
+	// Check that closing twice doesn't crash.
+	// The return value doesn't matter.
+	f.Close()
+
+	// Use iotest.TestReader to check small reads, Seek, ReadAt.
+	f, err = t.fsys.Open(file)
+	if err != nil {
+		t.errorf("%s: second Open: %v", file, err)
+		return
+	}
+	defer f.Close()
+	if err := iotest.TestReader(f, data); err != nil {
+		t.errorf("%s: failed TestReader:\n\t%s", file, strings.ReplaceAll(err.Error(), "\n", "\n\t"))
+	}
+}
+
+func (t *fsTester) checkFileRead(file, desc string, data1, data2 []byte) {
+	if string(data1) != string(data2) {
+		t.errorf("%s: %s: different data returned\n\t%q\n\t%q", file, desc, data1, data2)
+		return
+	}
+}
+
+// checkOpen checks that various invalid forms of file's name cannot be opened.
+func (t *fsTester) checkOpen(file string) {
+	bad := []string{
+		"/" + file,
+		file + "/.",
+	}
+	if file == "." {
+		bad = append(bad, "/")
+	}
+	if i := strings.Index(file, "/"); i >= 0 {
+		bad = append(bad,
+			file[:i]+"//"+file[i+1:],
+			file[:i]+"/./"+file[i+1:],
+			file[:i]+`\`+file[i+1:],
+			file[:i]+"/../"+file,
+		)
+	}
+	if i := strings.LastIndex(file, "/"); i >= 0 {
+		bad = append(bad,
+			file[:i]+"//"+file[i+1:],
+			file[:i]+"/./"+file[i+1:],
+			file[:i]+`\`+file[i+1:],
+			file+"/../"+file[i+1:],
+		)
+	}
+
+	for _, b := range bad {
+		if f, err := t.fsys.Open(b); err == nil {
+			f.Close()
+			t.errorf("%s: Open(%s) succeeded, want error", file, b)
+		}
+	}
+}