mirror of
https://github.com/golang/go
synced 2024-09-23 11:20:17 -06:00
os: implement CopyFS
Fixes #62484 Change-Id: I5d8950dedf86af48f42a641940b34e62aa2cddcb Reviewed-on: https://go-review.googlesource.com/c/go/+/558995 Auto-Submit: Ian Lance Taylor <iant@google.com> Reviewed-by: Ian Lance Taylor <iant@google.com> Reviewed-by: Dmitri Shuralyov <dmitshur@google.com> LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
This commit is contained in:
parent
8a0fbd75a5
commit
d9be60974b
1
api/next/62484.txt
Normal file
1
api/next/62484.txt
Normal file
@ -0,0 +1 @@
|
||||
pkg os, func CopyFS(string, fs.FS) error #62484
|
2
doc/next/6-stdlib/99-minor/os/62484.md
Normal file
2
doc/next/6-stdlib/99-minor/os/62484.md
Normal file
@ -0,0 +1,2 @@
|
||||
The [`CopyFS`](/os#CopyFS) function copies an [`io/fs.FS`](/io/fs#FS)
|
||||
into the local filesystem.
|
@ -5,6 +5,8 @@
|
||||
package os
|
||||
|
||||
import (
|
||||
"internal/safefilepath"
|
||||
"io"
|
||||
"io/fs"
|
||||
"sort"
|
||||
)
|
||||
@ -123,3 +125,61 @@ func ReadDir(name string) ([]DirEntry, error) {
|
||||
sort.Slice(dirs, func(i, j int) bool { return dirs[i].Name() < dirs[j].Name() })
|
||||
return dirs, err
|
||||
}
|
||||
|
||||
// CopyFS copies the file system fsys into the directory dir,
|
||||
// creating dir if necessary.
|
||||
//
|
||||
// Newly created directories and files have their default modes
|
||||
// where any bits from the file in fsys that are not part of the
|
||||
// standard read, write, and execute permissions will be zeroed
|
||||
// out, and standard read and write permissions are set for owner,
|
||||
// group, and others while retaining any existing execute bits from
|
||||
// the file in fsys.
|
||||
//
|
||||
// Symbolic links in fsys are not supported, a *PathError with Err set
|
||||
// to ErrInvalid is returned on symlink.
|
||||
//
|
||||
// Copying stops at and returns the first error encountered.
|
||||
func CopyFS(dir string, fsys fs.FS) error {
|
||||
return fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fpath, err := safefilepath.FromFS(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newPath := joinPath(dir, fpath)
|
||||
if d.IsDir() {
|
||||
return MkdirAll(newPath, 0777)
|
||||
}
|
||||
|
||||
// TODO(panjf2000): handle symlinks with the help of fs.ReadLinkFS
|
||||
// once https://go.dev/issue/49580 is done.
|
||||
// we also need safefilepath.IsLocal from https://go.dev/cl/564295.
|
||||
if !d.Type().IsRegular() {
|
||||
return &PathError{Op: "CopyFS", Path: path, Err: ErrInvalid}
|
||||
}
|
||||
|
||||
r, err := fsys.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer r.Close()
|
||||
info, err := r.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w, err := OpenFile(newPath, O_CREATE|O_TRUNC|O_WRONLY, 0666|info.Mode()&0777)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := io.Copy(w, r); err != nil {
|
||||
w.Close()
|
||||
return &PathError{Op: "Copy", Path: newPath, Err: err}
|
||||
}
|
||||
return w.Close()
|
||||
})
|
||||
}
|
||||
|
@ -5,6 +5,7 @@
|
||||
package os_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
@ -3030,35 +3031,44 @@ func TestOpenFileKeepsPermissions(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirFS(t *testing.T) {
|
||||
t.Parallel()
|
||||
func forceMFTUpdateOnWindows(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
|
||||
if runtime.GOOS != "windows" {
|
||||
return
|
||||
}
|
||||
|
||||
// On Windows, we force the MFT to update by reading the actual metadata from GetFileInformationByHandle and then
|
||||
// explicitly setting that. Otherwise it might get out of sync with FindFirstFile. See golang.org/issues/42637.
|
||||
if runtime.GOOS == "windows" {
|
||||
if err := filepath.WalkDir("./testdata/dirfs", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stat, err := Stat(path) // This uses GetFileInformationByHandle internally.
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stat.ModTime() == info.ModTime() {
|
||||
return nil
|
||||
}
|
||||
if err := Chtimes(path, stat.ModTime(), stat.ModTime()); err != nil {
|
||||
t.Log(err) // We only log, not die, in case the test directory is not writable.
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
if err := filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stat, err := Stat(path) // This uses GetFileInformationByHandle internally.
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stat.ModTime() == info.ModTime() {
|
||||
return nil
|
||||
}
|
||||
if err := Chtimes(path, stat.ModTime(), stat.ModTime()); err != nil {
|
||||
t.Log(err) // We only log, not die, in case the test directory is not writable.
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirFS(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
forceMFTUpdateOnWindows(t, "./testdata/dirfs")
|
||||
|
||||
fsys := DirFS("./testdata/dirfs")
|
||||
if err := fstest.TestFS(fsys, "a", "b", "dir/x"); err != nil {
|
||||
t.Fatal(err)
|
||||
@ -3358,3 +3368,223 @@ func TestRandomLen(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyFS(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Test with disk filesystem.
|
||||
forceMFTUpdateOnWindows(t, "./testdata/dirfs")
|
||||
fsys := DirFS("./testdata/dirfs")
|
||||
tmpDir := t.TempDir()
|
||||
if err := CopyFS(tmpDir, fsys); err != nil {
|
||||
t.Fatal("CopyFS:", err)
|
||||
}
|
||||
forceMFTUpdateOnWindows(t, tmpDir)
|
||||
tmpFsys := DirFS(tmpDir)
|
||||
if err := fstest.TestFS(tmpFsys, "a", "b", "dir/x"); err != nil {
|
||||
t.Fatal("TestFS:", err)
|
||||
}
|
||||
if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := fs.ReadFile(fsys, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newData, err := fs.ReadFile(tmpFsys, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !bytes.Equal(data, newData) {
|
||||
return errors.New("file " + path + " contents differ")
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal("comparing two directories:", err)
|
||||
}
|
||||
|
||||
// Test with memory filesystem.
|
||||
fsys = fstest.MapFS{
|
||||
"william": {Data: []byte("Shakespeare\n")},
|
||||
"carl": {Data: []byte("Gauss\n")},
|
||||
"daVinci": {Data: []byte("Leonardo\n")},
|
||||
"einstein": {Data: []byte("Albert\n")},
|
||||
"dir/newton": {Data: []byte("Sir Isaac\n")},
|
||||
}
|
||||
tmpDir = t.TempDir()
|
||||
if err := CopyFS(tmpDir, fsys); err != nil {
|
||||
t.Fatal("CopyFS:", err)
|
||||
}
|
||||
forceMFTUpdateOnWindows(t, tmpDir)
|
||||
tmpFsys = DirFS(tmpDir)
|
||||
if err := fstest.TestFS(tmpFsys, "william", "carl", "daVinci", "einstein", "dir/newton"); err != nil {
|
||||
t.Fatal("TestFS:", err)
|
||||
}
|
||||
if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := fs.ReadFile(fsys, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newData, err := fs.ReadFile(tmpFsys, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !bytes.Equal(data, newData) {
|
||||
return errors.New("file " + path + " contents differ")
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal("comparing two directories:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyFSWithSymlinks(t *testing.T) {
|
||||
// Test it with absolute and relative symlinks that point inside and outside the tree.
|
||||
testenv.MustHaveSymlink(t)
|
||||
|
||||
// Create a directory and file outside.
|
||||
tmpDir := t.TempDir()
|
||||
outsideDir, err := MkdirTemp(tmpDir, "copyfs")
|
||||
if err != nil {
|
||||
t.Fatalf("MkdirTemp: %v", err)
|
||||
}
|
||||
outsideFile := filepath.Join(outsideDir, "file.out.txt")
|
||||
|
||||
if err := WriteFile(outsideFile, []byte("Testing CopyFS outside"), 0644); err != nil {
|
||||
t.Fatalf("WriteFile: %v", err)
|
||||
}
|
||||
|
||||
// Create a directory and file inside.
|
||||
testDataDir, err := filepath.Abs("./testdata/")
|
||||
if err != nil {
|
||||
t.Fatalf("filepath.Abs: %v", err)
|
||||
}
|
||||
insideDir := filepath.Join(testDataDir, "copyfs")
|
||||
if err := Mkdir(insideDir, 0755); err != nil {
|
||||
t.Fatalf("Mkdir: %v", err)
|
||||
}
|
||||
defer RemoveAll(insideDir)
|
||||
insideFile := filepath.Join(insideDir, "file.in.txt")
|
||||
if err := WriteFile(insideFile, []byte("Testing CopyFS inside"), 0644); err != nil {
|
||||
t.Fatalf("WriteFile: %v", err)
|
||||
}
|
||||
|
||||
// Create directories for symlinks.
|
||||
linkInDir := filepath.Join(insideDir, "in_symlinks")
|
||||
if err := Mkdir(linkInDir, 0755); err != nil {
|
||||
t.Fatalf("Mkdir: %v", err)
|
||||
}
|
||||
linkOutDir := filepath.Join(insideDir, "out_symlinks")
|
||||
if err := Mkdir(linkOutDir, 0755); err != nil {
|
||||
t.Fatalf("Mkdir: %v", err)
|
||||
}
|
||||
|
||||
// First, we create the absolute symlink pointing outside.
|
||||
outLinkFile := filepath.Join(linkOutDir, "file.abs.out.link")
|
||||
if err := Symlink(outsideFile, outLinkFile); err != nil {
|
||||
t.Fatalf("Symlink: %v", err)
|
||||
}
|
||||
|
||||
// Then, we create the relative symlink pointing outside.
|
||||
relOutsideFile, err := filepath.Rel(filepath.Join(linkOutDir, "."), outsideFile)
|
||||
if err != nil {
|
||||
t.Fatalf("filepath.Rel: %v", err)
|
||||
}
|
||||
relOutLinkFile := filepath.Join(linkOutDir, "file.rel.out.link")
|
||||
if err := Symlink(relOutsideFile, relOutLinkFile); err != nil {
|
||||
t.Fatalf("Symlink: %v", err)
|
||||
}
|
||||
|
||||
// Last, we create the relative symlink pointing inside.
|
||||
relInsideFile, err := filepath.Rel(filepath.Join(linkInDir, "."), insideFile)
|
||||
if err != nil {
|
||||
t.Fatalf("filepath.Rel: %v", err)
|
||||
}
|
||||
relInLinkFile := filepath.Join(linkInDir, "file.rel.in.link")
|
||||
if err := Symlink(relInsideFile, relInLinkFile); err != nil {
|
||||
t.Fatalf("Symlink: %v", err)
|
||||
}
|
||||
|
||||
// Copy the directory tree and verify.
|
||||
forceMFTUpdateOnWindows(t, insideDir)
|
||||
fsys := DirFS(insideDir)
|
||||
tmpDupDir, err := MkdirTemp(tmpDir, "copyfs_dup")
|
||||
if err != nil {
|
||||
t.Fatalf("MkdirTemp: %v", err)
|
||||
}
|
||||
|
||||
// TODO(panjf2000): symlinks are currently not supported, and a specific error
|
||||
// will be returned. Verify that error and skip the subsequent test,
|
||||
// revisit this once #49580 is closed.
|
||||
if err := CopyFS(tmpDupDir, fsys); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatalf("got %v, want ErrInvalid", err)
|
||||
}
|
||||
t.Skip("skip the subsequent test and wait for #49580")
|
||||
|
||||
forceMFTUpdateOnWindows(t, tmpDupDir)
|
||||
tmpFsys := DirFS(tmpDupDir)
|
||||
if err := fstest.TestFS(tmpFsys, "file.in.txt", "out_symlinks/file.abs.out.link", "out_symlinks/file.rel.out.link", "in_symlinks/file.rel.in.link"); err != nil {
|
||||
t.Fatal("TestFS:", err)
|
||||
}
|
||||
if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
fi, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if filepath.Ext(path) == ".link" {
|
||||
if fi.Mode()&ModeSymlink == 0 {
|
||||
return errors.New("original file " + path + " should be a symlink")
|
||||
}
|
||||
tmpfi, err := fs.Stat(tmpFsys, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tmpfi.Mode()&ModeSymlink != 0 {
|
||||
return errors.New("copied file " + path + " should not be a symlink")
|
||||
}
|
||||
}
|
||||
|
||||
data, err := fs.ReadFile(fsys, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newData, err := fs.ReadFile(tmpFsys, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !bytes.Equal(data, newData) {
|
||||
return errors.New("file " + path + " contents differ")
|
||||
}
|
||||
|
||||
var target string
|
||||
switch fileName := filepath.Base(path); fileName {
|
||||
case "file.abs.out.link", "file.rel.out.link":
|
||||
target = outsideFile
|
||||
case "file.rel.in.link":
|
||||
target = insideFile
|
||||
}
|
||||
if len(target) > 0 {
|
||||
targetData, err := ReadFile(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !bytes.Equal(targetData, newData) {
|
||||
return errors.New("file " + path + " contents differ from target")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal("comparing two directories:", err)
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user