dashboard / erock/pico / feat(storage): base storage fn #70 rss

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

diff --git a/pkg/shared/storage/base.go b/pkg/shared/storage/base.go
new file mode 100644
index 0000000..a6690ff
--- /dev/null
+++ b/pkg/shared/storage/base.go
@@ -0,0 +1,34 @@
+package storage
+
+import (
+	"fmt"
+	"log/slog"
+
+	"github.com/picosh/utils"
+)
+
+func GetStorageTypeFromEnv() string {
+	return utils.GetEnv("STORAGE_ADAPTER", "minio")
+}
+
+func NewStorage(logger *slog.Logger, adapter string) (StorageServe, error) {
+	logger.Info("storage adapter", "adapter", adapter)
+	switch adapter {
+	case "minio":
+		minioURL := utils.GetEnv("MINIO_URL", "")
+		minioUser := utils.GetEnv("MINIO_ROOT_USER", "")
+		minioPass := utils.GetEnv("MINIO_ROOT_PASSWORD", "")
+		logger.Info("using minio storage", "url", minioURL, "user", minioUser)
+		return NewStorageMinio(logger, minioURL, minioUser, minioPass)
+	case "fs":
+		fsPath := utils.GetEnv("FS_STORAGE_DIR", "/tmp/pico_storage")
+		logger.Info("using filesystem storage", "path", fsPath)
+		return NewStorageFS(logger, fsPath)
+	case "memory":
+		data := map[string]map[string]string{}
+		logger.Info("using memory storage")
+		return NewStorageMemory(data)
+	default:
+		return nil, fmt.Errorf("unsupported storage type: %s", adapter)
+	}
+}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

diff --git a/Dockerfile b/Dockerfile
index 66acb7e..e85f0d2 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -54,7 +54,7 @@ ARG APP=prose
 
 COPY --from=builder-web /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=builder-web /go/bin/${APP}-web ./web
-COPY --from=builder-web /app/pkg/apps/${APP}/html ./${APP}/html
+COPY --from=builder-web /app/pkg/apps/${APP}/html ./pkg/apps/${APP}/html
 COPY --from=builder-web /app/pkg/apps/${APP}/public ./${APP}/public
 
 ENTRYPOINT ["/app/web"]
@@ -69,7 +69,7 @@ ARG APP=prose
 COPY --from=builder-ssh /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=builder-ssh /go/bin/${APP}-ssh ./ssh
 # some services require the html folder
-COPY --from=builder-ssh /app/pkg/apps/${APP}/html ./${APP}/html
+COPY --from=builder-ssh /app/pkg/apps/${APP}/html ./pkg/apps/${APP}/html
 
 
 ENTRYPOINT ["/app/ssh"]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

diff --git a/pkg/apps/prose/api.go b/pkg/apps/prose/api.go
index a94dc40..a933e9c 100644
--- a/pkg/apps/prose/api.go
+++ b/pkg/apps/prose/api.go
@@ -931,16 +931,11 @@ func StartApiServer() {
 	}()
 	logger := cfg.Logger
 
-	var st storage.StorageServe
-	var err error
-	if cfg.MinioURL == "" {
-		st, err = storage.NewStorageFS(cfg.Logger, cfg.StorageDir)
-	} else {
-		st, err = storage.NewStorageMinio(cfg.Logger, cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
-	}
-
+	adapter := storage.GetStorageTypeFromEnv()
+	st, err := storage.NewStorage(cfg.Logger, adapter)
 	if err != nil {
-		logger.Error(err.Error())
+		logger.Error("loading storage", "err", err)
+		return
 	}
 
 	staticRoutes := createStaticRoutes()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

diff --git a/pkg/apps/prose/ssh.go b/pkg/apps/prose/ssh.go
index 0a8ad6e..9c7945c 100644
--- a/pkg/apps/prose/ssh.go
+++ b/pkg/apps/prose/ssh.go
@@ -44,16 +44,10 @@ func StartSshServer() {
 		Db:  dbh,
 	}
 
-	var st storage.StorageServe
-	var err error
-	if cfg.MinioURL == "" {
-		st, err = storage.NewStorageFS(cfg.Logger, cfg.StorageDir)
-	} else {
-		st, err = storage.NewStorageMinio(cfg.Logger, cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
-	}
-
+	adapter := storage.GetStorageTypeFromEnv()
+	st, err := storage.NewStorage(cfg.Logger, adapter)
 	if err != nil {
-		logger.Error("storage", "err", err.Error())
+		logger.Error("loading storage", "err", err)
 		return
 	}
 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

diff --git a/pkg/shared/config.go b/pkg/shared/config.go
index 9352ff4..37fac51 100644
--- a/pkg/shared/config.go
+++ b/pkg/shared/config.go
@@ -122,7 +122,7 @@ func (c *ConfigSite) ReadURL() string {
 }
 
 func (c *ConfigSite) StaticPath(fname string) string {
-	return path.Join(c.Space, fname)
+	return path.Join("pkg", "apps", c.Space, fname)
 }
 
 func (c *ConfigSite) BlogURL(username string) string {

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

diff --git a/docker-compose.override.yml b/docker-compose.override.yml
index fa1a839..030d387 100644
--- a/docker-compose.override.yml
+++ b/docker-compose.override.yml
@@ -15,11 +15,9 @@ services:
     env_file:
       - .env.example
     volumes:
-      - ./data/storage/data:/storage
+      - /tmp/pico_storage:/storage
     ports:
       - "8080:8080"
-    links:
-      - minio
   pastes-web:
     build:
       args:

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101

diff --git a/pkg/apps/pgs/web.go b/pkg/apps/pgs/web.go
index e0b409f..431c330 100644
--- a/pkg/apps/pgs/web.go
+++ b/pkg/apps/pgs/web.go
@@ -439,7 +439,7 @@ func (web *WebRouter) ImageRequest(perm func(proj *db.Project) bool) http.Handle
 	}
 }
 
-func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, fromImgs bool, hasPerm HasPerm, w http.ResponseWriter, r *http.Request) {
+func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, fromProse bool, hasPerm HasPerm, w http.ResponseWriter, r *http.Request) {
 	subdomain := shared.GetSubdomain(r)
 
 	logger := web.Cfg.Logger.With(
@@ -459,6 +459,11 @@ func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, fro
 		return
 	}
 
+	// override for requests from prose
+	if fromProse {
+		props.ProjectName = "prose"
+	}
+
 	logger = logger.With(
 		"project", props.ProjectName,
 		"user", props.Username,
@@ -475,41 +480,30 @@ func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, fro
 		"userId", user.ID,
 	)
 
-	projectID := ""
-	// TODO: this could probably be cleaned up more
-	// imgs wont have a project directory
-	projectDir := ""
 	var bucket sst.Bucket
-	// imgs has a different bucket directory
-	if fromImgs {
-		bucket, err = web.Cfg.Storage.GetBucket(shared.GetImgsBucketName(user.ID))
-	} else {
-		bucket, err = web.Cfg.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
-		project, perr := web.Cfg.DB.FindProjectByName(user.ID, props.ProjectName)
-		if perr != nil {
-			logger.Info("project not found")
-			http.Error(w, "project not found", http.StatusNotFound)
-			return
-		}
+	bucket, err = web.Cfg.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
+	project, perr := web.Cfg.DB.FindProjectByName(user.ID, props.ProjectName)
+	if perr != nil {
+		logger.Info("project not found")
+		http.Error(w, "project not found", http.StatusNotFound)
+		return
+	}
 
-		logger = logger.With(
-			"projectId", project.ID,
-			"project", project.Name,
-		)
+	logger = logger.With(
+		"projectId", project.ID,
+		"project", project.Name,
+	)
 
-		if project.Blocked != "" {
-			logger.Error("project has been blocked")
-			http.Error(w, project.Blocked, http.StatusForbidden)
-			return
-		}
+	if project.Blocked != "" {
+		logger.Error("project has been blocked")
+		http.Error(w, project.Blocked, http.StatusForbidden)
+		return
+	}
 
-		projectID = project.ID
-		projectDir = project.ProjectDir
-		if !hasPerm(project) {
-			logger.Error("You do not have access to this site")
-			http.Error(w, "You do not have access to this site", http.StatusUnauthorized)
-			return
-		}
+	if !hasPerm(project) {
+		logger.Error("You do not have access to this site")
+		http.Error(w, "You do not have access to this site", http.StatusUnauthorized)
+		return
 	}
 
 	if err != nil {
@@ -533,11 +527,11 @@ func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, fro
 		Username:       props.Username,
 		UserID:         user.ID,
 		Subdomain:      subdomain,
-		ProjectDir:     projectDir,
+		ProjectID:      project.ID,
+		ProjectDir:     project.ProjectDir,
 		Filepath:       fname,
 		Bucket:         bucket,
 		ImgProcessOpts: opts,
-		ProjectID:      projectID,
 		HasPicoPlus:    hasPicoPlus,
 	}
 

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104

diff --git a/pkg/apps/prose/api.go b/pkg/apps/prose/api.go
index a933e9c..6fd85aa 100644
--- a/pkg/apps/prose/api.go
+++ b/pkg/apps/prose/api.go
@@ -4,10 +4,11 @@ import (
 	"bytes"
 	"fmt"
 	"html/template"
+	"io"
 	"net/http"
-	"net/http/httputil"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -865,6 +866,7 @@ func createMainRoutes(staticRoutes []shared.Route) []shared.Route {
 
 func imgRequest(w http.ResponseWriter, r *http.Request) {
 	logger := shared.GetLogger(r)
+	st := shared.GetStorage(r)
 	dbpool := shared.GetDB(r)
 	username := shared.GetUsernameFromRequest(r)
 	user, err := dbpool.FindUserByName(username)
@@ -875,22 +877,67 @@ func imgRequest(w http.ResponseWriter, r *http.Request) {
 	}
 	logger = shared.LoggerWithUser(logger, user)
 
-	destUrl, err := url.Parse(fmt.Sprintf("https://%s-prose.pgs.sh%s", username, r.URL.Path))
+	rawname := shared.GetField(r, 0)
+	imgOpts := shared.GetField(r, 1)
+	fname := filepath.Join("/prose", rawname)
+
+	opts, err := storage.UriToImgProcessOpts(imgOpts)
+	if err != nil {
+		errMsg := fmt.Sprintf("error processing img options: %s", err.Error())
+		logger.Error("error processing img options", "err", errMsg)
+		http.Error(w, errMsg, http.StatusUnprocessableEntity)
+		return
+	}
+
+	bucket, err := st.GetBucket(shared.GetAssetBucketName(user.ID))
 	if err != nil {
-		logger.Error("could not parse image proxy url", "username", username)
-		http.Error(w, "could not parse image proxy url", http.StatusInternalServerError)
+		logger.Error("bucket", "err", err)
+		http.Error(w, err.Error(), http.StatusUnprocessableEntity)
 		return
 	}
-	logger.Info("proxy image request", "url", destUrl.String())
 
-	proxy := httputil.NewSingleHostReverseProxy(destUrl)
-	oldDirector := proxy.Director
-	proxy.Director = func(r *http.Request) {
-		oldDirector(r)
-		r.Host = destUrl.Host
-		r.URL = destUrl
+	fmt.Println("HERERERERER", fname, opts)
+	contents, info, err := st.ServeObject(bucket, fname, opts)
+	if err != nil {
+		logger.Error("serve object", "err", err)
+		http.Error(w, err.Error(), http.StatusUnprocessableEntity)
+		return
+	}
+
+	contentType := ""
+	if info != nil {
+		contentType = info.Metadata.Get("content-type")
+		if info.Size != 0 {
+			w.Header().Add("content-length", strconv.Itoa(int(info.Size)))
+		}
+		if info.ETag != "" {
+			// Minio SDK trims off the mandatory quotes (RFC 7232 § 2.3)
+			w.Header().Add("etag", fmt.Sprintf("\"%s\"", info.ETag))
+		}
+
+		if !info.LastModified.IsZero() {
+			w.Header().Add("last-modified", info.LastModified.UTC().Format(http.TimeFormat))
+		}
+	}
+
+	if w.Header().Get("content-type") == "" {
+		w.Header().Set("content-type", contentType)
+	}
+
+	// Allows us to invalidate the cache when files are modified
+	// w.Header().Set("surrogate-key", h.Subdomain)
+
+	finContentType := w.Header().Get("content-type")
+	logger.Info(
+		"serving asset",
+		"asset", fname,
+		"contentType", finContentType,
+	)
+
+	_, err = io.Copy(w, contents)
+	if err != nil {
+		logger.Error("io copy", "err", err)
 	}
-	proxy.ServeHTTP(w, r)
 }
 
 func createSubdomainRoutes(staticRoutes []shared.Route) []shared.Route {

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

diff --git a/pkg/shared/storage/fs.go b/pkg/shared/storage/fs.go
index 0c75fcc..38d2826 100644
--- a/pkg/shared/storage/fs.go
+++ b/pkg/shared/storage/fs.go
@@ -39,7 +39,7 @@ func (s *StorageFS) ServeObject(bucket sst.Bucket, fpath string, opts *ImgProces
 		info.Metadata.Set("content-type", mimeType)
 	} else {
 		filePath := filepath.Join(bucket.Name, fpath)
-		dataURL := fmt.Sprintf("s3://%s", filePath)
+		dataURL := fmt.Sprintf("local:///%s", filePath)
 		rc, info, err = HandleProxy(s.Logger, dataURL, opts)
 	}
 	if err != nil {

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

diff --git a/pkg/apps/prose/api.go b/pkg/apps/prose/api.go
index 6fd85aa..0db8b20 100644
--- a/pkg/apps/prose/api.go
+++ b/pkg/apps/prose/api.go
@@ -897,7 +897,7 @@ func imgRequest(w http.ResponseWriter, r *http.Request) {
 	}
 
 	fmt.Println("HERERERERER", fname, opts)
-	contents, info, err := st.ServeObject(bucket, fname, opts)
+	contents, info, err := st.ServeObject(r, bucket, fname, opts)
 	if err != nil {
 		logger.Error("serve object", "err", err)
 		http.Error(w, err.Error(), http.StatusUnprocessableEntity)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

diff --git a/pkg/shared/storage/fs.go b/pkg/shared/storage/fs.go
index 38d2826..a8ce0ce 100644
--- a/pkg/shared/storage/fs.go
+++ b/pkg/shared/storage/fs.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
@@ -25,7 +26,7 @@ func NewStorageFS(logger *slog.Logger, dir string) (*StorageFS, error) {
 	return &StorageFS{st, logger}, nil
 }
 
-func (s *StorageFS) ServeObject(bucket sst.Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, *sst.ObjectInfo, error) {
+func (s *StorageFS) ServeObject(r *http.Request, bucket sst.Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, *sst.ObjectInfo, error) {
 	var rc io.ReadCloser
 	info := &sst.ObjectInfo{}
 	var err error
@@ -40,7 +41,7 @@ func (s *StorageFS) ServeObject(bucket sst.Bucket, fpath string, opts *ImgProces
 	} else {
 		filePath := filepath.Join(bucket.Name, fpath)
 		dataURL := fmt.Sprintf("local:///%s", filePath)
-		rc, info, err = HandleProxy(s.Logger, dataURL, opts)
+		rc, info, err = HandleProxy(r, s.Logger, dataURL, opts)
 	}
 	if err != nil {
 		return nil, nil, err

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

diff --git a/pkg/shared/storage/memory.go b/pkg/shared/storage/memory.go
index 7c3df6e..5a5c358 100644
--- a/pkg/shared/storage/memory.go
+++ b/pkg/shared/storage/memory.go
@@ -21,7 +21,7 @@ func NewStorageMemory(sto map[string]map[string]string) (*StorageMemory, error)
 	return &StorageMemory{st}, nil
 }
 
-func (s *StorageMemory) ServeObject(bucket sst.Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, *sst.ObjectInfo, error) {
+func (s *StorageMemory) ServeObject(r *http.Request, bucket sst.Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, *sst.ObjectInfo, error) {
 	obj, info, err := s.GetObject(bucket, fpath)
 	if info.Metadata == nil {
 		info.Metadata = make(http.Header)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

diff --git a/pkg/shared/storage/minio.go b/pkg/shared/storage/minio.go
index 5826fe6..d549372 100644
--- a/pkg/shared/storage/minio.go
+++ b/pkg/shared/storage/minio.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
@@ -24,7 +25,7 @@ func NewStorageMinio(logger *slog.Logger, address, user, pass string) (*StorageM
 	return &StorageMinio{st}, nil
 }
 
-func (s *StorageMinio) ServeObject(bucket sst.Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, *sst.ObjectInfo, error) {
+func (s *StorageMinio) ServeObject(r *http.Request, bucket sst.Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, *sst.ObjectInfo, error) {
 	var rc io.ReadCloser
 	info := &sst.ObjectInfo{}
 	var err error
@@ -39,7 +40,7 @@ func (s *StorageMinio) ServeObject(bucket sst.Bucket, fpath string, opts *ImgPro
 	} else {
 		filePath := filepath.Join(bucket.Name, fpath)
 		dataURL := fmt.Sprintf("s3://%s", filePath)
-		rc, info, err = HandleProxy(s.Logger, dataURL, opts)
+		rc, info, err = HandleProxy(r, s.Logger, dataURL, opts)
 	}
 	if err != nil {
 		return nil, nil, err

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

diff --git a/pkg/shared/storage/proxy.go b/pkg/shared/storage/proxy.go
index 9ad295b..27d516b 100644
--- a/pkg/shared/storage/proxy.go
+++ b/pkg/shared/storage/proxy.go
@@ -133,7 +133,7 @@ func (img *ImgProcessOpts) String() string {
 	return processOpts
 }
 
-func HandleProxy(logger *slog.Logger, dataURL string, opts *ImgProcessOpts) (io.ReadCloser, *storage.ObjectInfo, error) {
+func HandleProxy(r *http.Request, logger *slog.Logger, dataURL string, opts *ImgProcessOpts) (io.ReadCloser, *storage.ObjectInfo, error) {
 	imgProxyURL := os.Getenv("IMGPROXY_URL")
 	imgProxySalt := os.Getenv("IMGPROXY_SALT")
 	imgProxyKey := os.Getenv("IMGPROXY_KEY")
@@ -162,7 +162,16 @@ func HandleProxy(logger *slog.Logger, dataURL string, opts *ImgProcessOpts) (io.
 	}
 	proxyAddress := fmt.Sprintf("%s/%s%s", imgProxyURL, signature, processPath)
 
-	res, err := http.Get(proxyAddress)
+	req, err := http.NewRequest(http.MethodGet, proxyAddress, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	req.Header.Set("accept", r.Header.Get("accept"))
+	req.Header.Set("accept-encoding", r.Header.Get("accept-encoding"))
+	req.Header.Set("accept-language", r.Header.Get("accept-language"))
+	req.Header.Set("content-type", r.Header.Get("content-type"))
+	fmt.Println("HEADERS", req.Header)
+	res, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, nil, err
 	}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

diff --git a/pkg/shared/storage/storage.go b/pkg/shared/storage/storage.go
index 1a1b0f6..b1a1a82 100644
--- a/pkg/shared/storage/storage.go
+++ b/pkg/shared/storage/storage.go
@@ -2,11 +2,12 @@ package storage
 
 import (
 	"io"
+	"net/http"
 
 	sst "github.com/picosh/pico/pkg/pobj/storage"
 )
 
 type StorageServe interface {
 	sst.ObjectStorage
-	ServeObject(bucket sst.Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, *sst.ObjectInfo, error)
+	ServeObject(r *http.Request, bucket sst.Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, *sst.ObjectInfo, error)
 }