pico

created pr with ps-94 on 2025-01-18T20:18:40Z · by c8ef7d19
added ps-95 on 2025-01-18T20:20:30Z · by c8ef7d19
1: 89e0a31 ! 1: aa126f8 chore(prose): migrate images to pgs
changed status to accepted on 2025-02-01T18:43:36Z · by c8ef7d19
cmds
checkout latest patchset:
ssh pr.pico.sh print pr-44 | git am -3
checkout any patchset in a patch request:
ssh pr.pico.sh print ps-X | git am -3
add changes to patch request:
git format-patch main --stdout | ssh pr.pico.sh pr add 44
set PR to open (enables RSS notifications):
ssh pr.pico.sh pr open 44
set PR to draft (stops RSS notifications):
ssh pr.pico.sh pr draft 44
+82 -0 cmd/scripts/prose-imgs-migrate/main.go #
@@ -0,0 +1,82 @@
+package main
+
+import (
+	"bytes"
+	"io"
+	"log/slog"
+	"path/filepath"
+	"time"
+
+	"github.com/picosh/pico/db"
+	"github.com/picosh/pico/db/postgres"
+	"github.com/picosh/pico/prose"
+	"github.com/picosh/pico/shared"
+	"github.com/picosh/pico/shared/storage"
+	sst "github.com/picosh/pobj/storage"
+	sendUtils "github.com/picosh/send/utils"
+)
+
+func bail(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func upload(logger *slog.Logger, st storage.StorageServe, bucket sst.Bucket, fpath string, rdr io.Reader) error {
+	toSite := filepath.Join("prose", fpath)
+	logger.Info("uploading object", "bucket", bucket.Name, "object", toSite)
+	buf := &bytes.Buffer{}
+	size, err := io.Copy(buf, rdr)
+	if err != nil {
+		return err
+	}
+
+	_, _, err = st.PutObject(bucket, toSite, buf, &sendUtils.FileEntry{
+		Mtime: time.Now().Unix(),
+		Size:  size,
+	})
+	return err
+}
+
+func images(logger *slog.Logger, st storage.StorageServe, bucket sst.Bucket, user *db.User) error {
+	imgBucket, err := st.GetBucket(shared.GetImgsBucketName(user.ID))
+	if err != nil {
+		logger.Info("user does not have an images dir, skipping")
+		return nil
+	}
+	imgs, err := st.ListObjects(imgBucket, "/", false)
+	if err != nil {
+		return err
+	}
+
+	for _, inf := range imgs {
+		rdr, _, err := st.GetObject(imgBucket, inf.Name())
+		if err != nil {
+			return err
+		}
+		err = upload(logger, st, bucket, inf.Name(), rdr)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func main() {
+	cfg := prose.NewConfigSite()
+	logger := cfg.Logger
+	picoDb := postgres.NewDB(cfg.DbURL, logger)
+	st, err := storage.NewStorageMinio(logger, cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
+	bail(err)
+
+	users, err := picoDb.FindUsers()
+	bail(err)
+
+	for _, user := range users {
+		bucket, err := st.UpsertBucket(shared.GetAssetBucketName(user.ID))
+		bail(err)
+		_, _ = picoDb.InsertProject(user.ID, "prose", "prose")
+		bail(images(logger, st, bucket, user))
+	}
+}
+8 -4 filehandlers/imgs/handler.go #
@@ -47,6 +47,10 @@ func NewUploadImgHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage.S
 	}
 }
 
+func (h *UploadImgHandler) getObjectPath(fpath string) string {
+	return filepath.Join("prose", fpath)
+}
+
 func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.FileInfo, sendutils.ReaderAtCloser, error) {
 	user, err := h.DBPool.FindUser(s.Permissions().Extensions["user_id"])
 	if err != nil {
@@ -71,12 +75,12 @@ func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.F
 		FModTime: *post.UpdatedAt,
 	}
 
-	bucket, err := h.Storage.GetBucket(user.ID)
+	bucket, err := h.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
 	if err != nil {
 		return nil, nil, err
 	}
 
-	contents, _, err := h.Storage.GetObject(bucket, post.Filename)
+	contents, _, err := h.Storage.GetObject(bucket, h.getObjectPath(post.Filename))
 	if err != nil {
 		return nil, nil, err
 	}
@@ -218,13 +222,13 @@ func (h *UploadImgHandler) Delete(s ssh.Session, entry *sendutils.FileEntry) err
 		return fmt.Errorf("error for %s: %v", filename, err)
 	}
 
-	bucket, err := h.Storage.UpsertBucket(user.ID)
+	bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(user.ID))
 	if err != nil {
 		return err
 	}
 
 	logger.Info("deleting image")
-	err = h.Storage.DeleteObject(bucket, filename)
+	err = h.Storage.DeleteObject(bucket, h.getObjectPath(filename))
 	if err != nil {
 		return err
 	}
+2 -24 filehandlers/imgs/img.go #
@@ -49,7 +49,7 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
 		return nil
 	}
 
-	bucket, err := h.Storage.UpsertBucket(data.User.ID)
+	bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(data.User.ID))
 	if err != nil {
 		return err
 	}
@@ -58,7 +58,7 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
 
 	fname, _, err := h.Storage.PutObject(
 		bucket,
-		data.Filename,
+		h.getObjectPath(data.Filename),
 		sendutils.NopReaderAtCloser(reader),
 		&sendutils.FileEntry{},
 	)
@@ -128,18 +128,6 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error {
 			logger.Error("post could not create", "err", err.Error())
 			return fmt.Errorf("error for %s: %v", data.Filename, err)
 		}
-
-		if len(data.Tags) > 0 {
-			logger.Info(
-				"found post tags, replacing with old tags",
-				"tags", strings.Join(data.Tags, ","),
-			)
-			err = h.DBPool.ReplaceTagsForPost(data.Tags, data.Post.ID)
-			if err != nil {
-				logger.Error("post could not replace tags", "err", err.Error())
-				return fmt.Errorf("error for %s: %v", data.Filename, err)
-			}
-		}
 	} else {
 		if data.Shasum == data.Cur.Shasum && modTime.Equal(*data.Cur.UpdatedAt) {
 			logger.Info("image found, but image is identical, skipping")
@@ -167,16 +155,6 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error {
 			logger.Error("post could not update", "err", err.Error())
 			return fmt.Errorf("error for %s: %v", data.Filename, err)
 		}
-
-		logger.Info(
-			"found post tags, replacing with old tags",
-			"tags", strings.Join(data.Tags, ","),
-		)
-		err = h.DBPool.ReplaceTagsForPost(data.Tags, data.Cur.ID)
-		if err != nil {
-			logger.Error("post could not replace tags", "err", err.Error())
-			return fmt.Errorf("error for %s: %v", data.Filename, err)
-		}
 	}
 
 	return nil
+0 -168 imgs/api.go #
@@ -1,168 +0,0 @@
-package imgs
-
-import (
-	"fmt"
-	"html/template"
-	"net/http"
-	"net/url"
-	"path/filepath"
-
-	"github.com/picosh/pico/db"
-	"github.com/picosh/pico/pgs"
-	"github.com/picosh/pico/shared"
-	"github.com/picosh/pico/shared/storage"
-	"github.com/picosh/utils"
-)
-
-type PostPageData struct {
-	ImgURL template.URL
-}
-
-type BlogPageData struct {
-	Site      *shared.SitePageData
-	PageTitle string
-	URL       template.URL
-	Username  string
-	Posts     []template.URL
-}
-
-var Space = "imgs"
-
-func ImgsListHandler(w http.ResponseWriter, r *http.Request) {
-	username := shared.GetUsernameFromRequest(r)
-	dbpool := shared.GetDB(r)
-	logger := shared.GetLogger(r)
-	cfg := shared.GetCfg(r)
-
-	user, err := dbpool.FindUserForName(username)
-	if err != nil {
-		logger.Info("blog not found", "username", username)
-		http.Error(w, "blog not found", http.StatusNotFound)
-		return
-	}
-
-	var posts []*db.Post
-	pager := &db.Pager{Num: 1000, Page: 0}
-	p, err := dbpool.FindPostsForUser(pager, user.ID, Space)
-	posts = p.Data
-
-	if err != nil {
-		logger.Error(err.Error())
-		http.Error(w, "could not fetch posts for blog", http.StatusInternalServerError)
-		return
-	}
-
-	ts, err := shared.RenderTemplate(cfg, []string{
-		cfg.StaticPath("html/imgs.page.tmpl"),
-	})
-
-	if err != nil {
-		logger.Error(err.Error())
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-
-	curl := shared.CreateURLFromRequest(cfg, r)
-	postCollection := make([]template.URL, 0, len(posts))
-	for _, post := range posts {
-		url := cfg.ImgURL(curl, post.Username, post.Slug)
-		postCollection = append(postCollection, template.URL(url))
-	}
-
-	data := BlogPageData{
-		Site:      cfg.GetSiteData(),
-		PageTitle: fmt.Sprintf("%s imgs", username),
-		URL:       template.URL(cfg.FullBlogURL(curl, username)),
-		Username:  username,
-		Posts:     postCollection,
-	}
-
-	err = ts.Execute(w, data)
-	if err != nil {
-		logger.Error(err.Error())
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-	}
-}
-
-func anyPerm(proj *db.Project) bool {
-	return true
-}
-
-func ImgRequest(w http.ResponseWriter, r *http.Request) {
-	subdomain := shared.GetSubdomain(r)
-	cfg := shared.GetCfg(r)
-	st := shared.GetStorage(r)
-	dbpool := shared.GetDB(r)
-	logger := shared.GetLogger(r)
-	username := shared.GetUsernameFromRequest(r)
-
-	user, err := dbpool.FindUserForName(username)
-	if err != nil {
-		logger.Info("user not found", "user", username)
-		http.Error(w, "user not found", http.StatusNotFound)
-		return
-	}
-
-	var imgOpts string
-	var slug string
-	if !cfg.IsSubdomains() || subdomain == "" {
-		slug, _ = url.PathUnescape(shared.GetField(r, 1))
-		imgOpts, _ = url.PathUnescape(shared.GetField(r, 2))
-	} else {
-		slug, _ = url.PathUnescape(shared.GetField(r, 0))
-		imgOpts, _ = url.PathUnescape(shared.GetField(r, 1))
-	}
-
-	opts, err := storage.UriToImgProcessOpts(imgOpts)
-	if err != nil {
-		errMsg := fmt.Sprintf("error processing img options: %s", err.Error())
-		logger.Info(errMsg)
-		http.Error(w, errMsg, http.StatusUnprocessableEntity)
-		return
-	}
-
-	// set default quality for web optimization
-	if opts.Quality == 0 {
-		opts.Quality = 80
-	}
-
-	ext := filepath.Ext(slug)
-	// set default format to be webp
-	if opts.Ext == "" && ext == "" {
-		opts.Ext = "webp"
-	}
-
-	// Files can contain periods.  `filepath.Ext` is greedy and will clip the last period in the slug
-	// and call that a file extension so we want to be explicit about what
-	// file extensions we clip here
-	for _, fext := range cfg.AllowedExt {
-		if ext == fext {
-			// users might add the file extension when requesting an image
-			// but we want to remove that
-			slug = utils.SanitizeFileExt(slug)
-			break
-		}
-	}
-
-	post, err := FindImgPost(r, user, slug)
-	if err != nil {
-		errMsg := fmt.Sprintf("image not found %s/%s", user.Name, slug)
-		logger.Info(errMsg)
-		http.Error(w, errMsg, http.StatusNotFound)
-		return
-	}
-
-	fname := post.Filename
-	router := pgs.NewWebRouter(
-		cfg,
-		logger,
-		dbpool,
-		st,
-	)
-	router.ServeAsset(fname, opts, true, anyPerm, w, r)
-}
-
-func FindImgPost(r *http.Request, user *db.User, slug string) (*db.Post, error) {
-	dbpool := shared.GetDB(r)
-	return dbpool.FindPostWithSlug(slug, user.ID, Space)
-}
+0 -1 imgs/html/rss.page.tmpl #
@@ -1,1 +0,0 @@
-<img src="{{.ImgURL}}" />
+0 -0 imgs/public/.gitkeep #
+21 -12 prose/api.go #
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"html/template"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
 	"os"
 	"strconv"
@@ -16,7 +17,6 @@ import (
 	"github.com/gorilla/feeds"
 	"github.com/picosh/pico/db"
 	"github.com/picosh/pico/db/postgres"
-	"github.com/picosh/pico/imgs"
 	"github.com/picosh/pico/shared"
 	"github.com/picosh/pico/shared/storage"
 	"github.com/picosh/utils"
@@ -438,14 +438,6 @@ func postHandler(w http.ResponseWriter, r *http.Request) {
 			WithStyles:   withStyles,
 		}
 	} else {
-		// TODO: HACK to support imgs slugs inside prose
-		// We definitely want to kill this feature in time
-		imgPost, err := imgs.FindImgPost(r, user, slug)
-		if err == nil && imgPost != nil {
-			imgs.ImgRequest(w, r)
-			return
-		}
-
 		notFound, err := dbpool.FindPostWithFilename("_404.md", user.ID, cfg.Space)
 		contents := template.HTML("Oops!  we can't seem to find this post.")
 		title := "Post not found"
@@ -859,6 +851,24 @@ func createMainRoutes(staticRoutes []shared.Route) []shared.Route {
 	return routes
 }
 
+func imgRequest(w http.ResponseWriter, r *http.Request) {
+	username := shared.GetUsernameFromRequest(r)
+	destUrl, err := url.Parse(fmt.Sprintf("https://%s-prose.pgs.sh", username))
+	if err != nil {
+		http.Error(w, "site not found", http.StatusNotFound)
+		return
+	}
+
+	proxy := httputil.NewSingleHostReverseProxy(destUrl)
+	oldDirector := proxy.Director
+	proxy.Director = func(r *http.Request) {
+		oldDirector(r)
+		r.Host = destUrl.Host
+		r.URL = destUrl
+	}
+	proxy.ServeHTTP(w, r)
+}
+
 func createSubdomainRoutes(staticRoutes []shared.Route) []shared.Route {
 	routes := []shared.Route{
 		shared.NewRoute("GET", "/", blogHandler),
@@ -881,9 +891,8 @@ func createSubdomainRoutes(staticRoutes []shared.Route) []shared.Route {
 	routes = append(
 		routes,
 		shared.NewRoute("GET", "/raw/(.+)", postRawHandler),
-		shared.NewRoute("GET", "/([^/]+)/(.+)", imgs.ImgRequest),
-		shared.NewRoute("GET", "/(.+.(?:jpg|jpeg|png|gif|webp|svg))$", imgs.ImgRequest),
-		shared.NewRoute("GET", "/i", imgs.ImgsListHandler),
+		shared.NewRoute("GET", "/([^/]+)/(.+)", imgRequest),
+		shared.NewRoute("GET", "/(.+.(?:jpg|jpeg|png|gif|webp|svg))$", imgRequest),
 		shared.NewRoute("GET", "/(.+)", postHandler),
 	)
 
+3 -0 prose/ssh.go #
@@ -79,6 +79,9 @@ func StartSshServer() {
 		return
 	}
 
+	ctx := context.Background()
+	defer ctx.Done()
+
 	fileMap := map[string]filehandlers.ReadWriteHandler{
 		".md":      filehandlers.NewScpPostHandler(dbh, cfg, hooks),
 		".css":     filehandlers.NewScpPostHandler(dbh, cfg, hooks),
Back to top