pico

chore(pobj.fs): feature parity with minio rss

created pr with 66.1 on 2025-04-18T17:16:07Z · by c8ef7d19

changed pr name to chore(pobj.fs): feature parity with minio on 2025-04-18T17:18:11Z · by c8ef7d19

added 66.2 on 2025-04-21T18:44:44Z · by c8ef7d19


                      1:  a3f9ef2 = 1:  a3f9ef2 chore: rm UserMetadata


                      2:  ccd89b0 = 2:  ccd89b0 refactor(shared): mime type pkg


                      3:  8621e35 = 3:  8621e35 chore(pobj.fs): provide metadata


                      -:  ------- > 4:  831b150 chore(storage.fs): add tests

cmds

checkout latest patchset:

ssh pr.pico.sh print 66 | git am -3

checkout any patchset in a patch request:

ssh pr.pico.sh print 66.[rev] | git am -3

add changes to patch request:

git format-patch main --stdout | ssh pr.pico.sh pr add 66

Patchset `66.2` on 2025-04-21T18:44:44Z · commit 831b150

chore: rm UserMetadata

Eric Bower 2025-04-18T17:01:42Z

refactor(shared): mime type pkg

Eric Bower 2025-04-18T17:09:48Z

chore(pobj.fs): provide metadata

Eric Bower 2025-04-18T17:15:37Z

→ chore(storage.fs): add tests

Eric Bower 2025-04-21T18:43:56Z

chore(storage.fs): add tests

Eric Bower 2025-04-21T18:43:56Z

Semantic diff summary

1 added, 3 modified, 0 signature changed, 0 removed across 2 analyzed files

pkg/pobj/storage/fs.go

function_declaration UpsertBucket modified
function_declaration GetBucketQuota modified
method_declaration ListBuckets modified

pkg/shared/storage/fs_test.go

function_declaration TestFsAdapter added

+25 -4 pkg/pobj/storage/fs.go #

......@@ -76,7 +76,7 @@ func (s *StorageFS) UpsertBucket(name string) (Bucket, error) {
		return bucket, nil
	}

	dir := filepath.Join(s.Dir, bucket.Path)
	dir := filepath.Join(s.Dir, name)
	s.Logger.Info("bucket not found, creating", "dir", dir, "err", err)
	err = os.MkdirAll(dir, os.ModePerm)
	if err != nil {

......@@ -91,6 +91,8 @@ func (s *StorageFS) GetBucketQuota(bucket Bucket) (uint64, error) {
	return uint64(dsize), err
}

// DeleteBucket will delete all contents regardless if files exist inside of it.
// This is different from minio impl which requires all files be deleted first.
func (s *StorageFS) DeleteBucket(bucket Bucket) error {
	return os.RemoveAll(bucket.Path)
}

......@@ -150,15 +152,34 @@ func (s *StorageFS) DeleteObject(bucket Bucket, fpath string) error {
		return err
	}

	// try to remove dir if it is empty
	// traverse up the folder tree and remove all empty folders
	dir := filepath.Dir(loc)
	_ = os.Remove(dir)
	for dir != "" {
		err = os.Remove(dir)
		if err != nil {
			break
		}
		fp := strings.Split(dir, "/")
		dir = "/" + filepath.Join(fp[:len(fp)-1]...)
	}

	return nil
}

func (s *StorageFS) ListBuckets() ([]string, error) {
	return []string{}, fmt.Errorf("not implemented")
	entries, err := os.ReadDir(s.Dir)
	if err != nil {
		return []string{}, err
	}

	buckets := []string{}
	for _, e := range entries {
		if !e.IsDir() {
			continue
		}
		buckets = append(buckets, e.Name())
	}
	return buckets, nil
}

func (s *StorageFS) ListObjects(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error) {

+217 -0 pkg/shared/storage/fs_test.go #

......@@ -0,0 +1,217 @@
package storage

import (
	"io"
	"io/fs"
	"log/slog"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
	"github.com/picosh/pico/pkg/send/utils"
)

func TestFsAdapter(t *testing.T) {
	logger := slog.Default()
	f, err := os.MkdirTemp("", "fs-tests-")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(f)

	st, err := NewStorageFS(logger, f)
	if err != nil {
		t.Fatal(err)
	}

	bucketName := "main"
	// create bucket
	bucket, err := st.UpsertBucket(bucketName)
	if err != nil {
		t.Fatal(err)
	}

	// ensure bucket exists
	file, err := os.Stat(bucket.Path)
	if err != nil {
		t.Fatal(err)
	}
	if !file.IsDir() {
		t.Fatal("bucket must be directory")
	}

	bucketCheck, err := st.GetBucket(bucketName)
	if err != nil {
		t.Fatal(err)
	}
	if bucketCheck.Path != bucket.Path || bucketCheck.Name != bucket.Name {
		t.Fatal("upsert and get bucket incongruent")
	}

	modTime := time.Now()

	str := "here is a test file"
	reader := strings.NewReader(str)
	actualPath, size, err := st.PutObject(bucket, "./nice/test.txt", reader, &utils.FileEntry{
		Mtime: modTime.Unix(),
	})
	if err != nil {
		t.Fatal(err)
	}
	if size != int64(len(str)) {
		t.Fatalf("size, actual: %d, expected: %d", size, int64(len(str)))
	}
	expectedPath := filepath.Join(bucket.Path, "nice", "test.txt")
	if actualPath != expectedPath {
		t.Fatalf("path, actual: %s, expected: %s", actualPath, expectedPath)
	}

	// ensure file exists
	_, err = os.Stat(expectedPath)
	if err != nil {
		t.Fatal(err)
	}

	// get file
	r, info, err := st.GetObject(bucket, "nice/test.txt")
	if err != nil {
		t.Fatal(err)
	}
	buf := new(strings.Builder)
	_, err = io.Copy(buf, r)
	if err != nil {
		t.Fatal(err)
	}
	actualStr := buf.String()
	if actualStr != str {
		t.Fatalf("contents, actual: %s, expected: %s", actualStr, str)
	}
	if info.Size != size {
		t.Fatalf("size, actual: %d, expected: %d", size, info.Size)
	}

	str = "a deeply nested test file"
	reader = strings.NewReader(str)
	_, _, err = st.PutObject(bucket, "./here/we/go/again.txt", reader, &utils.FileEntry{
		Mtime: modTime.Unix(),
	})
	if err != nil {
		t.Fatal(err)
	}

	// list objects
	objs, err := st.ListObjects(bucket, "/", true)
	if err != nil {
		t.Fatal(err)
	}

	expectedObjs := []fs.FileInfo{
		&utils.VirtualFile{
			FName:  "main",
			FIsDir: true,
			FSize:  80,
		},
		&utils.VirtualFile{
			FName:  "here",
			FIsDir: true,
			FSize:  60,
		},
		&utils.VirtualFile{
			FName:  "we",
			FIsDir: true,
			FSize:  60,
		},
		&utils.VirtualFile{
			FName:  "go",
			FIsDir: true,
			FSize:  60,
		},
		&utils.VirtualFile{FName: "again.txt", FSize: 25},
		&utils.VirtualFile{
			FName:  "nice",
			FIsDir: true,
			FSize:  60,
		},
		&utils.VirtualFile{FName: "test.txt", FSize: 19},
	}
	ignore := cmpopts.IgnoreFields(utils.VirtualFile{}, "FModTime")
	if cmp.Equal(objs, expectedObjs, ignore) == false {
		//nolint
		t.Fatal(cmp.Diff(objs, expectedObjs, ignore))
	}

	// list buckets
	aBucket, _ := st.UpsertBucket("another")
	_, _ = st.UpsertBucket("and-another")
	buckets, err := st.ListBuckets()
	if err != nil {
		t.Fatal(err)
	}
	expectedBuckets := []string{"and-another", "another", "main"}
	if cmp.Equal(buckets, expectedBuckets) == false {
		//nolint
		t.Fatal(cmp.Diff(buckets, expectedBuckets))
	}

	// delete bucket
	err = st.DeleteBucket(aBucket)
	if err != nil {
		t.Fatal(err)
	}

	// ensure bucket was actually deleted
	_, err = os.Stat(aBucket.Path)
	if !os.IsNotExist(err) {
		t.Fatal("directory should have been deleted")
	}

	err = st.DeleteObject(bucket, "nice/test.txt")
	if err != nil {
		t.Fatal(err)
	}

	// ensure file was actually deleted
	_, err = os.Stat(filepath.Join(bucket.Path, "nice/test.txt"))
	if !os.IsNotExist(err) {
		t.Fatal("file should have been deleted")
	}

	// ensure containing folder was also deleted
	_, err = os.Stat(filepath.Join(bucket.Path, "nice"))
	if !os.IsNotExist(err) {
		t.Fatal("containing folder should have been deleted")
	}

	str = "a deeply nested test file"
	reader = strings.NewReader(str)
	_, _, err = st.PutObject(bucket, "./here/yes/we/can.txt", reader, &utils.FileEntry{
		Mtime: modTime.Unix(),
	})
	if err != nil {
		t.Fatal(err)
	}

	// delete deeply nested file and all parent folders that are now empty
	err = st.DeleteObject(bucket, "here/yes/we/can.txt")
	if err != nil {
		t.Fatal(err)
	}
	_, err = os.Stat(filepath.Join(bucket.Path, "here"))
	if os.IsNotExist(err) {
		t.Fatal("this folder had multiple files and should not have been deleted")
	}
	_, err = os.Stat(filepath.Join(bucket.Path, "here/yes"))
	if !os.IsNotExist(err) {
		t.Fatal("containing folder should have been deleted")
	}

	// delete bucket even with file contents
	err = st.DeleteBucket(bucket)
	if err != nil {
		t.Fatal(err)
	}
}

...	...		@@ -91,6 +91,8 @@ func (s *StorageFS) GetBucketQuota(bucket Bucket) (uint64, error) {
91	91		return uint64(dsize), err
92	92		}
93	93
	94	+	// DeleteBucket will delete all contents regardless if files exist inside of it.
	95	+	// This is different from minio impl which requires all files be deleted first.
94	96		func (s *StorageFS) DeleteBucket(bucket Bucket) error {
95	97		return os.RemoveAll(bucket.Path)
96	98		}

...	...		@@ -150,15 +152,34 @@ func (s *StorageFS) DeleteObject(bucket Bucket, fpath string) error {
150	152		return err
151	153		}
152	154
153		-	// try to remove dir if it is empty
	155	+	// traverse up the folder tree and remove all empty folders
154	156		dir := filepath.Dir(loc)
155		-	_ = os.Remove(dir)
	157	+	for dir != "" {
	158	+	err = os.Remove(dir)
	159	+	if err != nil {
	160	+	break
	161	+	}
	162	+	fp := strings.Split(dir, "/")
	163	+	dir = "/" + filepath.Join(fp[:len(fp)-1]...)
	164	+	}
156	165
157	166		return nil
158	167		}
159	168
160	169		func (s *StorageFS) ListBuckets() ([]string, error) {
161		-	return []string{}, fmt.Errorf("not implemented")
	170	+	entries, err := os.ReadDir(s.Dir)
	171	+	if err != nil {
	172	+	return []string{}, err
	173	+	}
	174	+
	175	+	buckets := []string{}
	176	+	for _, e := range entries {
	177	+	if !e.IsDir() {
	178	+	continue
	179	+	}
	180	+	buckets = append(buckets, e.Name())
	181	+	}
	182	+	return buckets, nil
162	183		}
163	184
164	185		func (s *StorageFS) ListObjects(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error) {

...	...		@@ -76,7 +76,7 @@ func (s *StorageFS) UpsertBucket(name string) (Bucket, error) {
76	76		return bucket, nil
77	77		}
78	78
79		-	dir := filepath.Join(s.Dir, bucket.Path)
	79	+	dir := filepath.Join(s.Dir, name)
80	80		s.Logger.Info("bucket not found, creating", "dir", dir, "err", err)
81	81		err = os.MkdirAll(dir, os.ModePerm)
82	82		if err != nil {

...	...		@@ -0,0 +1,217 @@
	1	+	package storage
	2	+
	3	+	import (
	4	+	"io"
	5	+	"io/fs"
	6	+	"log/slog"
	7	+	"os"
	8	+	"path/filepath"
	9	+	"strings"
	10	+	"testing"
	11	+	"time"
	12	+
	13	+	"github.com/google/go-cmp/cmp"
	14	+	"github.com/google/go-cmp/cmp/cmpopts"
	15	+	"github.com/picosh/pico/pkg/send/utils"
	16	+	)
	17	+
	18	+	func TestFsAdapter(t *testing.T) {
	19	+	logger := slog.Default()
	20	+	f, err := os.MkdirTemp("", "fs-tests-")
	21	+	if err != nil {
	22	+	t.Fatal(err)
	23	+	}
	24	+	defer os.RemoveAll(f)
	25	+
	26	+	st, err := NewStorageFS(logger, f)
	27	+	if err != nil {
	28	+	t.Fatal(err)
	29	+	}
	30	+
	31	+	bucketName := "main"
	32	+	// create bucket
	33	+	bucket, err := st.UpsertBucket(bucketName)
	34	+	if err != nil {
	35	+	t.Fatal(err)
	36	+	}
	37	+
	38	+	// ensure bucket exists
	39	+	file, err := os.Stat(bucket.Path)
	40	+	if err != nil {
	41	+	t.Fatal(err)
	42	+	}
	43	+	if !file.IsDir() {
	44	+	t.Fatal("bucket must be directory")
	45	+	}
	46	+
	47	+	bucketCheck, err := st.GetBucket(bucketName)
	48	+	if err != nil {
	49	+	t.Fatal(err)
	50	+	}
	51	+	if bucketCheck.Path != bucket.Path \|\| bucketCheck.Name != bucket.Name {
	52	+	t.Fatal("upsert and get bucket incongruent")
	53	+	}
	54	+
	55	+	modTime := time.Now()
	56	+
	57	+	str := "here is a test file"
	58	+	reader := strings.NewReader(str)
	59	+	actualPath, size, err := st.PutObject(bucket, "./nice/test.txt", reader, &utils.FileEntry{
	60	+	Mtime: modTime.Unix(),
	61	+	})
	62	+	if err != nil {
	63	+	t.Fatal(err)
	64	+	}
	65	+	if size != int64(len(str)) {
	66	+	t.Fatalf("size, actual: %d, expected: %d", size, int64(len(str)))
	67	+	}
	68	+	expectedPath := filepath.Join(bucket.Path, "nice", "test.txt")
	69	+	if actualPath != expectedPath {
	70	+	t.Fatalf("path, actual: %s, expected: %s", actualPath, expectedPath)
	71	+	}
	72	+
	73	+	// ensure file exists
	74	+	_, err = os.Stat(expectedPath)
	75	+	if err != nil {
	76	+	t.Fatal(err)
	77	+	}
	78	+
	79	+	// get file
	80	+	r, info, err := st.GetObject(bucket, "nice/test.txt")
	81	+	if err != nil {
	82	+	t.Fatal(err)
	83	+	}
	84	+	buf := new(strings.Builder)
	85	+	_, err = io.Copy(buf, r)
	86	+	if err != nil {
	87	+	t.Fatal(err)
	88	+	}
	89	+	actualStr := buf.String()
	90	+	if actualStr != str {
	91	+	t.Fatalf("contents, actual: %s, expected: %s", actualStr, str)
	92	+	}
	93	+	if info.Size != size {
	94	+	t.Fatalf("size, actual: %d, expected: %d", size, info.Size)
	95	+	}
	96	+
	97	+	str = "a deeply nested test file"
	98	+	reader = strings.NewReader(str)
	99	+	_, _, err = st.PutObject(bucket, "./here/we/go/again.txt", reader, &utils.FileEntry{
	100	+	Mtime: modTime.Unix(),
	101	+	})
	102	+	if err != nil {
	103	+	t.Fatal(err)
	104	+	}
	105	+
	106	+	// list objects
	107	+	objs, err := st.ListObjects(bucket, "/", true)
	108	+	if err != nil {
	109	+	t.Fatal(err)
	110	+	}
	111	+
	112	+	expectedObjs := []fs.FileInfo{
	113	+	&utils.VirtualFile{
	114	+	FName: "main",
	115	+	FIsDir: true,
	116	+	FSize: 80,
	117	+	},
	118	+	&utils.VirtualFile{
	119	+	FName: "here",
	120	+	FIsDir: true,
	121	+	FSize: 60,
	122	+	},
	123	+	&utils.VirtualFile{
	124	+	FName: "we",
	125	+	FIsDir: true,
	126	+	FSize: 60,
	127	+	},
	128	+	&utils.VirtualFile{
	129	+	FName: "go",
	130	+	FIsDir: true,
	131	+	FSize: 60,
	132	+	},
	133	+	&utils.VirtualFile{FName: "again.txt", FSize: 25},
	134	+	&utils.VirtualFile{
	135	+	FName: "nice",
	136	+	FIsDir: true,
	137	+	FSize: 60,
	138	+	},
	139	+	&utils.VirtualFile{FName: "test.txt", FSize: 19},
	140	+	}
	141	+	ignore := cmpopts.IgnoreFields(utils.VirtualFile{}, "FModTime")
	142	+	if cmp.Equal(objs, expectedObjs, ignore) == false {
	143	+	//nolint
	144	+	t.Fatal(cmp.Diff(objs, expectedObjs, ignore))
	145	+	}
	146	+
	147	+	// list buckets
	148	+	aBucket, _ := st.UpsertBucket("another")
	149	+	_, _ = st.UpsertBucket("and-another")
	150	+	buckets, err := st.ListBuckets()
	151	+	if err != nil {
	152	+	t.Fatal(err)
	153	+	}
	154	+	expectedBuckets := []string{"and-another", "another", "main"}
	155	+	if cmp.Equal(buckets, expectedBuckets) == false {
	156	+	//nolint
	157	+	t.Fatal(cmp.Diff(buckets, expectedBuckets))
	158	+	}
	159	+
	160	+	// delete bucket
	161	+	err = st.DeleteBucket(aBucket)
	162	+	if err != nil {
	163	+	t.Fatal(err)
	164	+	}
	165	+
	166	+	// ensure bucket was actually deleted
	167	+	_, err = os.Stat(aBucket.Path)
	168	+	if !os.IsNotExist(err) {
	169	+	t.Fatal("directory should have been deleted")
	170	+	}
	171	+
	172	+	err = st.DeleteObject(bucket, "nice/test.txt")
	173	+	if err != nil {
	174	+	t.Fatal(err)
	175	+	}
	176	+
	177	+	// ensure file was actually deleted
	178	+	_, err = os.Stat(filepath.Join(bucket.Path, "nice/test.txt"))
	179	+	if !os.IsNotExist(err) {
	180	+	t.Fatal("file should have been deleted")
	181	+	}
	182	+
	183	+	// ensure containing folder was also deleted
	184	+	_, err = os.Stat(filepath.Join(bucket.Path, "nice"))
	185	+	if !os.IsNotExist(err) {
	186	+	t.Fatal("containing folder should have been deleted")
	187	+	}
	188	+
	189	+	str = "a deeply nested test file"
	190	+	reader = strings.NewReader(str)
	191	+	_, _, err = st.PutObject(bucket, "./here/yes/we/can.txt", reader, &utils.FileEntry{
	192	+	Mtime: modTime.Unix(),
	193	+	})
	194	+	if err != nil {
	195	+	t.Fatal(err)
	196	+	}
	197	+
	198	+	// delete deeply nested file and all parent folders that are now empty
	199	+	err = st.DeleteObject(bucket, "here/yes/we/can.txt")
	200	+	if err != nil {
	201	+	t.Fatal(err)
	202	+	}
	203	+	_, err = os.Stat(filepath.Join(bucket.Path, "here"))
	204	+	if os.IsNotExist(err) {
	205	+	t.Fatal("this folder had multiple files and should not have been deleted")
	206	+	}
	207	+	_, err = os.Stat(filepath.Join(bucket.Path, "here/yes"))
	208	+	if !os.IsNotExist(err) {
	209	+	t.Fatal("containing folder should have been deleted")
	210	+	}
	211	+
	212	+	// delete bucket even with file contents
	213	+	err = st.DeleteBucket(bucket)
	214	+	if err != nil {
	215	+	t.Fatal(err)
	216	+	}
	217	+	}

pico

Patchset 66.2 on 2025-04-21T18:44:44Z · commit 831b150

Patchset `66.2` on 2025-04-21T18:44:44Z · commit 831b150