git-pr

refactor: custom index page open rss
created pr with 56.1 on 2025-03-27T20:16:09Z · by c8ef7d19
added 56.2 on 2025-03-27T20:17:48Z · by c8ef7d19
1: 0200c93 ! 1: a2710a3 refactor: custom index page
added 56.3 on 2025-03-28T14:48:26Z · by c8ef7d19
1: a2710a3 < -: ------- refactor: custom index page
-: ------- > 1: 7338b44 feat: allow config `desc` to add a description box to index page
added 56.4 on 2025-04-06T19:08:12Z · by c8ef7d19
1: 7338b44 < -: ------- feat: allow config `desc` to add a description box to index page
-: ------- > 1: 26daea4 feat(pgs): lru cache for object info and special files
-: ------- > 2: b004b64 chore(pgs): use http cache clear event to rm lru cache for special files
-: ------- > 3: 59f5618 refactor(pgs): store lru cache on web router
changed status to accepted on 2025-04-06T22:13:51Z · by c8ef7d19
cmds
checkout latest patchset: 
ssh pr.pico.sh print 56 | git am -3
checkout any patchset in a patch request: 
ssh pr.pico.sh print 56.[rev] | git am -3
add changes to patch request: 
git format-patch main --stdout | ssh pr.pico.sh pr add 56
set PR to open (enables RSS notifications): 
ssh pr.pico.sh pr open 56
set PR to draft (stops RSS notifications): 
ssh pr.pico.sh pr draft 56

Patchset 56.4 on 2025-04-06T19:08:12Z · commit 26daea4

→ feat(pgs): lru cache for object info and special files
Eric Bower 2025-04-06T02:55:19Z
 chore(pgs): use http cache clear event to rm lru cache for special files
Eric Bower 2025-04-06T03:03:27Z
 refactor(pgs): store lru cache on web router
Eric Bower 2025-04-06T19:07:36Z
+1 -0 go.mod #
......@@ -35,6 +35,7 @@ require (
3535 github.com/google/uuid v1.6.0
3636 github.com/gorilla/feeds v1.2.0
3737 github.com/gorilla/websocket v1.5.3
38+ github.com/hashicorp/golang-lru/v2 v2.0.7
3839 github.com/jmoiron/sqlx v1.4.0
3940 github.com/lib/pq v1.10.9
4041 github.com/matryer/is v1.4.1
+2 -0 go.sum #
......@@ -451,6 +451,8 @@ github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
451451 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
452452 github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
453453 github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
454+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
455+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
454456 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
455457 github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y=
456458 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+5 -5 pkg/apps/pgs/uploader.go #
......@@ -395,7 +395,7 @@ func (h *UploadAssetHandler) Write(s *pssh.SSHServerConnSession, entry *sendutil
395395 )
396396
397397 specialFileMax := featureFlag.Data.SpecialFileMax
398- if isSpecialFile(entry) {
398+ if isSpecialFile(entry.Filepath) {
399399 sizeRemaining = min(sizeRemaining, specialFileMax)
400400 }
401401
......@@ -441,9 +441,9 @@ func (h *UploadAssetHandler) Write(s *pssh.SSHServerConnSession, entry *sendutil
441441 return str, err
442442 }
443443
444-func isSpecialFile(entry *sendutils.FileEntry) bool {
445- fname := filepath.Base(entry.Filepath)
446- return fname == "_headers" || fname == "_redirects"
444+func isSpecialFile(entry string) bool {
445+ fname := filepath.Base(entry)
446+ return fname == "_headers" || fname == "_redirects" || fname == "_pgs_ignore"
447447 }
448448
449449 func (h *UploadAssetHandler) Delete(s *pssh.SSHServerConnSession, entry *sendutils.FileEntry) error {
......@@ -525,7 +525,7 @@ func (h *UploadAssetHandler) validateAsset(data *FileData) (bool, error) {
525525 }
526526
527527 // special files we use for custom routing
528- if fname == "_pgs_ignore" || fname == "_redirects" || fname == "_headers" {
528+ if isSpecialFile(fname) {
529529 return true, nil
530530 }
531531
+2 -1 pkg/apps/pgs/web.go #
......@@ -9,6 +9,7 @@ import (
99 "net/http"
1010 "net/url"
1111 "os"
12+ "path/filepath"
1213 "regexp"
1314 "strings"
1415 "time"
......@@ -426,7 +427,7 @@ func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, fro
426427 "host", r.Host,
427428 )
428429
429- if fname == "_headers" || fname == "_redirects" || fname == "_pgs_ignore" {
430+ if isSpecialFile(fname) {
430431 logger.Info("special file names are not allowed to be served over http")
431432 http.Error(w, "404 not found", http.StatusNotFound)
432433 return
+64 -41 pkg/apps/pgs/web_asset_handler.go #
......@@ -14,10 +14,17 @@ import (
1414 "net/http/httputil"
1515 _ "net/http/pprof"
1616
17+ "github.com/hashicorp/golang-lru/v2/expirable"
18+ "github.com/picosh/pico/pkg/cache"
1719 sst "github.com/picosh/pico/pkg/pobj/storage"
1820 "github.com/picosh/pico/pkg/shared/storage"
1921 )
2022
23+var (
24+ redirectsCache = expirable.NewLRU[string, []*RedirectRule](2048, nil, cache.CacheTimeout)
25+ headersCache = expirable.NewLRU[string, []*HeaderRule](2048, nil, cache.CacheTimeout)
26+)
27+
2128 type ApiAssetHandler struct {
2229 *WebRouter
2330 Logger *slog.Logger
......@@ -41,28 +48,36 @@ func hasProtocol(url string) bool {
4148 func (h *ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
4249 logger := h.Logger
4350 var redirects []*RedirectRule
44- redirectFp, redirectInfo, err := h.Cfg.Storage.GetObject(h.Bucket, filepath.Join(h.ProjectDir, "_redirects"))
45- if err == nil {
46- defer redirectFp.Close()
47- if redirectInfo != nil && redirectInfo.Size > h.Cfg.MaxSpecialFileSize {
48- errMsg := fmt.Sprintf("_redirects file is too large (%d > %d)", redirectInfo.Size, h.Cfg.MaxSpecialFileSize)
49- logger.Error(errMsg)
50- http.Error(w, errMsg, http.StatusInternalServerError)
51- return
52- }
53- buf := new(strings.Builder)
54- lr := io.LimitReader(redirectFp, h.Cfg.MaxSpecialFileSize)
55- _, err := io.Copy(buf, lr)
56- if err != nil {
57- logger.Error("io copy", "err", err.Error())
58- http.Error(w, "cannot read _redirects file", http.StatusInternalServerError)
59- return
60- }
6151
62- redirects, err = parseRedirectText(buf.String())
63- if err != nil {
64- logger.Error("could not parse redirect text", "err", err.Error())
52+ redirectsCacheKey := filepath.Join(h.Bucket.Name, h.ProjectDir, "_redirects")
53+ if cachedRedirects, found := redirectsCache.Get(redirectsCacheKey); found {
54+ redirects = cachedRedirects
55+ } else {
56+ redirectFp, redirectInfo, err := h.Cfg.Storage.GetObject(h.Bucket, filepath.Join(h.ProjectDir, "_redirects"))
57+ if err == nil {
58+ defer redirectFp.Close()
59+ if redirectInfo != nil && redirectInfo.Size > h.Cfg.MaxSpecialFileSize {
60+ errMsg := fmt.Sprintf("_redirects file is too large (%d > %d)", redirectInfo.Size, h.Cfg.MaxSpecialFileSize)
61+ logger.Error(errMsg)
62+ http.Error(w, errMsg, http.StatusInternalServerError)
63+ return
64+ }
65+ buf := new(strings.Builder)
66+ lr := io.LimitReader(redirectFp, h.Cfg.MaxSpecialFileSize)
67+ _, err := io.Copy(buf, lr)
68+ if err != nil {
69+ logger.Error("io copy", "err", err.Error())
70+ http.Error(w, "cannot read _redirects file", http.StatusInternalServerError)
71+ return
72+ }
73+
74+ redirects, err = parseRedirectText(buf.String())
75+ if err != nil {
76+ logger.Error("could not parse redirect text", "err", err.Error())
77+ }
6578 }
79+
80+ redirectsCache.Add(redirectsCacheKey, redirects)
6681 }
6782
6883 routes := calcRoutes(h.ProjectDir, h.Filepath, redirects)
......@@ -163,28 +178,36 @@ func (h *ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
163178 defer contents.Close()
164179
165180 var headers []*HeaderRule
166- headersFp, headersInfo, err := h.Cfg.Storage.GetObject(h.Bucket, filepath.Join(h.ProjectDir, "_headers"))
167- if err == nil {
168- defer headersFp.Close()
169- if headersInfo != nil && headersInfo.Size > h.Cfg.MaxSpecialFileSize {
170- errMsg := fmt.Sprintf("_headers file is too large (%d > %d)", headersInfo.Size, h.Cfg.MaxSpecialFileSize)
171- logger.Error(errMsg)
172- http.Error(w, errMsg, http.StatusInternalServerError)
173- return
174- }
175- buf := new(strings.Builder)
176- lr := io.LimitReader(headersFp, h.Cfg.MaxSpecialFileSize)
177- _, err := io.Copy(buf, lr)
178- if err != nil {
179- logger.Error("io copy", "err", err.Error())
180- http.Error(w, "cannot read _headers file", http.StatusInternalServerError)
181- return
182- }
183181
184- headers, err = parseHeaderText(buf.String())
185- if err != nil {
186- logger.Error("could not parse header text", "err", err.Error())
182+ headersCacheKey := filepath.Join(h.Bucket.Name, h.ProjectDir, "_headers")
183+ if cachedHeaders, found := headersCache.Get(headersCacheKey); found {
184+ headers = cachedHeaders
185+ } else {
186+ headersFp, headersInfo, err := h.Cfg.Storage.GetObject(h.Bucket, filepath.Join(h.ProjectDir, "_headers"))
187+ if err == nil {
188+ defer headersFp.Close()
189+ if headersInfo != nil && headersInfo.Size > h.Cfg.MaxSpecialFileSize {
190+ errMsg := fmt.Sprintf("_headers file is too large (%d > %d)", headersInfo.Size, h.Cfg.MaxSpecialFileSize)
191+ logger.Error(errMsg)
192+ http.Error(w, errMsg, http.StatusInternalServerError)
193+ return
194+ }
195+ buf := new(strings.Builder)
196+ lr := io.LimitReader(headersFp, h.Cfg.MaxSpecialFileSize)
197+ _, err := io.Copy(buf, lr)
198+ if err != nil {
199+ logger.Error("io copy", "err", err.Error())
200+ http.Error(w, "cannot read _headers file", http.StatusInternalServerError)
201+ return
202+ }
203+
204+ headers, err = parseHeaderText(buf.String())
205+ if err != nil {
206+ logger.Error("could not parse header text", "err", err.Error())
207+ }
187208 }
209+
210+ headersCache.Add(headersCacheKey, headers)
188211 }
189212
190213 userHeaders := []*HeaderLine{}
......@@ -236,7 +259,7 @@ func (h *ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
236259 return
237260 }
238261 w.WriteHeader(status)
239- _, err = io.Copy(w, contents)
262+ _, err := io.Copy(w, contents)
240263
241264 if err != nil {
242265 logger.Error("io copy", "err", err.Error())
+21 -0 pkg/cache/cache.go #
......@@ -0,0 +1,21 @@
1+package cache
2+
3+import (
4+ "log/slog"
5+ "time"
6+
7+ "github.com/picosh/utils"
8+)
9+
10+var CacheTimeout time.Duration
11+
12+func init() {
13+ cacheDuration := utils.GetEnv("STORAGE_MINIO_CACHE_DURATION", "1m")
14+ duration, err := time.ParseDuration(cacheDuration)
15+ if err != nil {
16+ slog.Error("Invalid STORAGE_MINIO_CACHE_DURATION value, using default 1m", "error", err)
17+ duration = 1 * time.Minute
18+ }
19+
20+ CacheTimeout = duration
21+}
+58 -18 pkg/pobj/storage/minio.go #
......@@ -7,13 +7,16 @@ import (
77 "io"
88 "net/url"
99 "os"
10+ "path/filepath"
1011 "strconv"
1112 "strings"
1213 "time"
1314
15+ "github.com/hashicorp/golang-lru/v2/expirable"
1416 "github.com/minio/madmin-go/v3"
1517 "github.com/minio/minio-go/v7"
1618 "github.com/minio/minio-go/v7/pkg/credentials"
19+ "github.com/picosh/pico/pkg/cache"
1720 "github.com/picosh/pico/pkg/send/utils"
1821 )
1922
......@@ -22,8 +25,23 @@ type StorageMinio struct {
2225 Admin *madmin.AdminClient
2326 }
2427
25-var _ ObjectStorage = &StorageMinio{}
26-var _ ObjectStorage = (*StorageMinio)(nil)
28+type CachedBucket struct {
29+ Bucket
30+ Error error
31+}
32+
33+type CachedObjectInfo struct {
34+ *ObjectInfo
35+ Error error
36+}
37+
38+var (
39+ _ ObjectStorage = &StorageMinio{}
40+ _ ObjectStorage = (*StorageMinio)(nil)
41+
42+ bucketCache = expirable.NewLRU[string, CachedBucket](2048, nil, cache.CacheTimeout)
43+ objectInfoCache = expirable.NewLRU[string, CachedObjectInfo](2048, nil, cache.CacheTimeout)
44+)
2745
2846 func NewStorageMinio(address, user, pass string) (*StorageMinio, error) {
2947 endpoint, err := url.Parse(address)
......@@ -59,6 +77,10 @@ func NewStorageMinio(address, user, pass string) (*StorageMinio, error) {
5977 }
6078
6179 func (s *StorageMinio) GetBucket(name string) (Bucket, error) {
80+ if cachedBucket, found := bucketCache.Get(name); found {
81+ return cachedBucket.Bucket, cachedBucket.Error
82+ }
83+
6284 bucket := Bucket{
6385 Name: name,
6486 }
......@@ -68,9 +90,13 @@ func (s *StorageMinio) GetBucket(name string) (Bucket, error) {
6890 if err == nil {
6991 err = errors.New("bucket does not exist")
7092 }
93+
94+ bucketCache.Add(name, CachedBucket{bucket, err})
7195 return bucket, err
7296 }
7397
98+ bucketCache.Add(name, CachedBucket{bucket, nil})
99+
74100 return bucket, nil
75101 }
76102
......@@ -160,29 +186,43 @@ func (s *StorageMinio) GetObject(bucket Bucket, fpath string) (utils.ReadAndRead
160186 ETag: "",
161187 }
162188
163- info, err := s.Client.StatObject(context.Background(), bucket.Name, fpath, minio.StatObjectOptions{})
164- if err != nil {
165- return nil, objInfo, err
166- }
189+ cacheKey := filepath.Join(bucket.Name, fpath)
190+
191+ cachedInfo, found := objectInfoCache.Get(cacheKey)
192+ if found {
193+ objInfo = cachedInfo.ObjectInfo
167194
168- objInfo.LastModified = info.LastModified
169- objInfo.ETag = info.ETag
170- objInfo.Metadata = info.Metadata
171- objInfo.UserMetadata = info.UserMetadata
172- objInfo.Size = info.Size
195+ if cachedInfo.Error != nil {
196+ return nil, objInfo, cachedInfo.Error
197+ }
198+ } else {
199+ info, err := s.Client.StatObject(context.Background(), bucket.Name, fpath, minio.StatObjectOptions{})
200+ if err != nil {
201+ objectInfoCache.Add(cacheKey, CachedObjectInfo{objInfo, err})
202+ return nil, objInfo, err
203+ }
204+
205+ objInfo.LastModified = info.LastModified
206+ objInfo.ETag = info.ETag
207+ objInfo.Metadata = info.Metadata
208+ objInfo.UserMetadata = info.UserMetadata
209+ objInfo.Size = info.Size
210+
211+ if mtime, ok := info.UserMetadata["Mtime"]; ok {
212+ mtimeUnix, err := strconv.Atoi(mtime)
213+ if err == nil {
214+ objInfo.LastModified = time.Unix(int64(mtimeUnix), 0)
215+ }
216+ }
217+
218+ objectInfoCache.Add(cacheKey, CachedObjectInfo{objInfo, nil})
219+ }
173220
174221 obj, err := s.Client.GetObject(context.Background(), bucket.Name, fpath, minio.GetObjectOptions{})
175222 if err != nil {
176223 return nil, objInfo, err
177224 }
178225
179- if mtime, ok := info.UserMetadata["Mtime"]; ok {
180- mtimeUnix, err := strconv.Atoi(mtime)
181- if err == nil {
182- objInfo.LastModified = time.Unix(int64(mtimeUnix), 0)
183- }
184- }
185-
186226 return obj, objInfo, nil
187227 }
188228
+5 -0 pkg/shared/storage/proxy.go #
......@@ -166,6 +166,7 @@ type ImgProcessOpts struct {
166166 Ratio *Ratio
167167 Rotate int
168168 Ext string
169+ NoRaw bool
169170 }
170171
171172 func (img *ImgProcessOpts) String() string {
......@@ -204,6 +205,10 @@ func (img *ImgProcessOpts) String() string {
204205 processOpts = fmt.Sprintf("%s/ext:%s", processOpts, img.Ext)
205206 }
206207
208+ if processOpts == "" && !img.NoRaw {
209+ processOpts = fmt.Sprintf("%s/raw:true", processOpts)
210+ }
211+
207212 return processOpts
208213 }
209214
Back to top