dashboard / erock/pico / feat(pgs): lru cache for object info and special files #59 rss

old

new

 		HeadersCache:   expirable.NewLRU[string, []*HeaderRule](2048, nil, cache.CacheTimeout),
 	}
 	router.initRouters()
+	go router.watchCacheClear()
 	return router
 }
 
+func (web *WebRouter) watchCacheClear() {
+	for key := range web.Cfg.CacheClearingQueue {
+		web.Cfg.Logger.Info("lru cache clear request", "key", key)
+		rKey := filepath.Join(key, "_redirects")
+		web.RedirectsCache.Remove(rKey)
+		hKey := filepath.Join(key, "_headers")
+		web.HeadersCache.Remove(hKey)
+	}
+}
+
 func (web *WebRouter) initRouters() {
 	// ensure legacy router is disabled
 	// GODEBUG=httpmuxgo121=0
 		}
 	}
 
-	go func() {
-		for key := range web.Cfg.CacheClearingQueue {
-			rKey := filepath.Join(key, "_redirects")
-			web.RedirectsCache.Remove(rKey)
-			hKey := filepath.Join(key, "_headers")
-			web.HeadersCache.Remove(hKey)
-		}
-	}()
-
 	asset := &ApiAssetHandler{
 		WebRouter: web,
 		Logger:    logger,

old

new

 	var redirects []*RedirectRule
 
 	redirectsCacheKey := filepath.Join(getSurrogateKey(h.UserID, h.ProjectDir), "_redirects")
+	logger.Info("looking for _redirects in lru cache", "key", redirectsCacheKey)
 	if cachedRedirects, found := h.RedirectsCache.Get(redirectsCacheKey); found {
-		fmt.Println(cachedRedirects)
+		logger.Info("_redirects found in lru cache", "key", redirectsCacheKey)
 		redirects = cachedRedirects
 	} else {
+		logger.Info("_redirects not found in lru cache", "key", redirectsCacheKey)
 		redirectFp, redirectInfo, err := h.Cfg.Storage.GetObject(h.Bucket, filepath.Join(h.ProjectDir, "_redirects"))
 		if err == nil {
 			defer redirectFp.Close()
 	var headers []*HeaderRule
 
 	headersCacheKey := filepath.Join(getSurrogateKey(h.UserID, h.ProjectDir), "_headers")
+	logger.Info("looking for _headers in lru cache", "key", headersCacheKey)
 	if cachedHeaders, found := h.HeadersCache.Get(headersCacheKey); found {
+		logger.Info("_headers found in lru", "key", headersCacheKey)
 		headers = cachedHeaders
 	} else {
+		logger.Info("_headers not found in lru cache", "key", headersCacheKey)
 		headersFp, headersInfo, err := h.Cfg.Storage.GetObject(h.Bucket, filepath.Join(h.ProjectDir, "_headers"))
 		if err == nil {
 			defer headersFp.Close()

old

new

 	"errors"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/url"
 	"os"
 	"path/filepath"
 )
 
 type StorageMinio struct {
-	Client *minio.Client
-	Admin  *madmin.AdminClient
+	Client      *minio.Client
+	Admin       *madmin.AdminClient
+	BucketCache *expirable.LRU[string, CachedBucket]
+	Logger      *slog.Logger
 }
 
 type CachedBucket struct {
 var (
 	_ ObjectStorage = &StorageMinio{}
 	_ ObjectStorage = (*StorageMinio)(nil)
-
-	bucketCache     = expirable.NewLRU[string, CachedBucket](2048, nil, cache.CacheTimeout)
-	objectInfoCache = expirable.NewLRU[string, CachedObjectInfo](2048, nil, cache.CacheTimeout)
 )
 
-func NewStorageMinio(address, user, pass string) (*StorageMinio, error) {
+func NewStorageMinio(logger *slog.Logger, address, user, pass string) (*StorageMinio, error) {
 	endpoint, err := url.Parse(address)
 	if err != nil {
 		return nil, err
 	}
 
 	mini := &StorageMinio{
-		Client: mClient,
-		Admin:  aClient,
+		Client:      mClient,
+		Admin:       aClient,
+		BucketCache: expirable.NewLRU[string, CachedBucket](2048, nil, cache.CacheTimeout),
+		Logger:      logger,
 	}
 	return mini, err
 }
 
 func (s *StorageMinio) GetBucket(name string) (Bucket, error) {
-	if cachedBucket, found := bucketCache.Get(name); found {
+	if cachedBucket, found := s.BucketCache.Get(name); found {
+		s.Logger.Info("bucket found in lru cache", "name", name)
 		return cachedBucket.Bucket, cachedBucket.Error
 	}
+	s.Logger.Info("bucket not found in lru cache", "name", name)
 
 	bucket := Bucket{
 		Name: name,
 			err = errors.New("bucket does not exist")
 		}
 
-		bucketCache.Add(name, CachedBucket{bucket, err})
+		s.BucketCache.Add(name, CachedBucket{bucket, err})
 		return bucket, err
 	}
 
-	bucketCache.Add(name, CachedBucket{bucket, nil})
+	s.BucketCache.Add(name, CachedBucket{bucket, nil})
 
 	return bucket, nil
 }
 
 	cacheKey := filepath.Join(bucket.Name, fpath)
 
-	cachedInfo, found := objectInfoCache.Get(cacheKey)
-	if found {
-		objInfo = cachedInfo.ObjectInfo
-
-		if cachedInfo.Error != nil {
-			return nil, objInfo, cachedInfo.Error
-		}
-	} else {
-		info, err := s.Client.StatObject(context.Background(), bucket.Name, fpath, minio.StatObjectOptions{})
-		if err != nil {
-			objectInfoCache.Add(cacheKey, CachedObjectInfo{objInfo, err})
-			return nil, objInfo, err
-		}
+	s.Logger.Info("object info not found in lru cache", "key", cacheKey)
+	info, err := s.Client.StatObject(context.Background(), bucket.Name, fpath, minio.StatObjectOptions{})
+	if err != nil {
+		return nil, objInfo, err
+	}
 
-		objInfo.LastModified = info.LastModified
-		objInfo.ETag = info.ETag
-		objInfo.Metadata = info.Metadata
-		objInfo.UserMetadata = info.UserMetadata
-		objInfo.Size = info.Size
+	objInfo.LastModified = info.LastModified
+	objInfo.ETag = info.ETag
+	objInfo.Metadata = info.Metadata
+	objInfo.UserMetadata = info.UserMetadata
+	objInfo.Size = info.Size
 
-		if mtime, ok := info.UserMetadata["Mtime"]; ok {
-			mtimeUnix, err := strconv.Atoi(mtime)
-			if err == nil {
-				objInfo.LastModified = time.Unix(int64(mtimeUnix), 0)
-			}
+	if mtime, ok := info.UserMetadata["Mtime"]; ok {
+		mtimeUnix, err := strconv.Atoi(mtime)
+		if err == nil {
+			objInfo.LastModified = time.Unix(int64(mtimeUnix), 0)
 		}
-
-		objectInfoCache.Add(cacheKey, CachedObjectInfo{objInfo, nil})
 	}
 
 	obj, err := s.Client.GetObject(context.Background(), bucket.Name, fpath, minio.GetObjectOptions{})

old

new

 			"url", url,
 			"user", user,
 		)
-		return storage.NewStorageMinio(url, user, pass)
+		return storage.NewStorageMinio(logger, url, user, pass)
 	}
 
 	// implied driver == "fs"

old

new

 
 type StorageMinio struct {
 	*sst.StorageMinio
-	Logger *slog.Logger
 }
 
 func NewStorageMinio(logger *slog.Logger, address, user, pass string) (*StorageMinio, error) {
-	st, err := sst.NewStorageMinio(address, user, pass)
+	st, err := sst.NewStorageMinio(logger, address, user, pass)
 	if err != nil {
 		return nil, err
 	}
-	return &StorageMinio{st, logger}, nil
+	return &StorageMinio{st}, nil
 }
 
 func (s *StorageMinio) ServeObject(bucket sst.Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, *sst.ObjectInfo, error) {