pico
created pr with
ps-94
added ps-95
1: 89e0a31 ! 1: aa126f8 chore(prose): migrate images to pgs
changed status to
accepted
cmds
checkout latest patchset:
ssh pr.pico.sh print pr-44 | git am -3checkout any patchset in a patch request:
ssh pr.pico.sh print ps-X | git am -3add changes to patch request:
git format-patch main --stdout | ssh pr.pico.sh pr add 44set PR to open (enables RSS notifications):
ssh pr.pico.sh pr open 44set PR to draft (stops RSS notifications):
ssh pr.pico.sh pr draft 44
Patchset
ps-94
chore(prose): migrate images to pgs
Eric Bower
2025-01-18T14:27:41ZSemantic diff summary
6 added,
9 modified,
0 signature changed,
6 removed
across 6 analyzed files
(2 files skipped: unsupported file type)
+82
-0
cmd/scripts/prose-imgs-migrate/main.go
#
@@ -0,0 +1,82 @@
+package main
+
+import (
+ "bytes"
+ "io"
+ "log/slog"
+ "path/filepath"
+ "time"
+
+ "github.com/picosh/pico/db"
+ "github.com/picosh/pico/db/postgres"
+ "github.com/picosh/pico/prose"
+ "github.com/picosh/pico/shared"
+ "github.com/picosh/pico/shared/storage"
+ sst "github.com/picosh/pobj/storage"
+ sendUtils "github.com/picosh/send/utils"
+)
+
+func bail(err error) {
+ if err != nil {
+ panic(err)
+ }
+}
+
+func upload(logger *slog.Logger, st storage.StorageServe, bucket sst.Bucket, fpath string, rdr io.Reader) error {
+ toSite := filepath.Join("prose", fpath)
+ logger.Info("uploading object", "bucket", bucket.Name, "object", toSite)
+ buf := &bytes.Buffer{}
+ size, err := io.Copy(buf, rdr)
+ if err != nil {
+ return err
+ }
+
+ _, _, err = st.PutObject(bucket, toSite, buf, &sendUtils.FileEntry{
+ Mtime: time.Now().Unix(),
+ Size: size,
+ })
+ return err
+}
+
+func images(logger *slog.Logger, st storage.StorageServe, bucket sst.Bucket, user *db.User) error {
+ imgBucket, err := st.GetBucket(shared.GetImgsBucketName(user.ID))
+ if err != nil {
+ logger.Info("user does not have an images dir, skipping")
+ return nil
+ }
+ imgs, err := st.ListObjects(imgBucket, "/", false)
+ if err != nil {
+ return err
+ }
+
+ for _, inf := range imgs {
+ rdr, _, err := st.GetObject(imgBucket, inf.Name())
+ if err != nil {
+ return err
+ }
+ err = upload(logger, st, bucket, inf.Name(), rdr)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func main() {
+ cfg := prose.NewConfigSite()
+ logger := cfg.Logger
+ picoDb := postgres.NewDB(cfg.DbURL, logger)
+ st, err := storage.NewStorageMinio(logger, cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
+ bail(err)
+
+ users, err := picoDb.FindUsers()
+ bail(err)
+
+ for _, user := range users {
+ bucket, err := st.UpsertBucket(shared.GetAssetBucketName(user.ID))
+ bail(err)
+ _, _ = picoDb.InsertProject(user.ID, "prose", "prose")
+ bail(images(logger, st, bucket, user))
+ }
+}
+8
-4
filehandlers/imgs/handler.go
#
@@ -47,6 +47,10 @@ func NewUploadImgHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage.S
}
}
+func (h *UploadImgHandler) getObjectPath(fpath string) string {
+ return filepath.Join("prose", fpath)
+}
+
func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.FileInfo, sendutils.ReaderAtCloser, error) {
user, err := h.DBPool.FindUser(s.Permissions().Extensions["user_id"])
if err != nil {
@@ -71,12 +75,12 @@ func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.F
FModTime: *post.UpdatedAt,
}
- bucket, err := h.Storage.GetBucket(user.ID)
+ bucket, err := h.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
if err != nil {
return nil, nil, err
}
- contents, _, err := h.Storage.GetObject(bucket, post.Filename)
+ contents, _, err := h.Storage.GetObject(bucket, h.getObjectPath(post.Filename))
if err != nil {
return nil, nil, err
}
@@ -218,13 +222,13 @@ func (h *UploadImgHandler) Delete(s ssh.Session, entry *sendutils.FileEntry) err
return fmt.Errorf("error for %s: %v", filename, err)
}
- bucket, err := h.Storage.UpsertBucket(user.ID)
+ bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(user.ID))
if err != nil {
return err
}
logger.Info("deleting image")
- err = h.Storage.DeleteObject(bucket, filename)
+ err = h.Storage.DeleteObject(bucket, h.getObjectPath(filename))
if err != nil {
return err
}
+2
-24
filehandlers/imgs/img.go
#
@@ -49,7 +49,7 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
return nil
}
- bucket, err := h.Storage.UpsertBucket(data.User.ID)
+ bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(data.User.ID))
if err != nil {
return err
}
@@ -58,7 +58,7 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
fname, _, err := h.Storage.PutObject(
bucket,
- data.Filename,
+ h.getObjectPath(data.Filename),
sendutils.NopReaderAtCloser(reader),
&sendutils.FileEntry{},
)
@@ -128,18 +128,6 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error {
logger.Error("post could not create", "err", err.Error())
return fmt.Errorf("error for %s: %v", data.Filename, err)
}
-
- if len(data.Tags) > 0 {
- logger.Info(
- "found post tags, replacing with old tags",
- "tags", strings.Join(data.Tags, ","),
- )
- err = h.DBPool.ReplaceTagsForPost(data.Tags, data.Post.ID)
- if err != nil {
- logger.Error("post could not replace tags", "err", err.Error())
- return fmt.Errorf("error for %s: %v", data.Filename, err)
- }
- }
} else {
if data.Shasum == data.Cur.Shasum && modTime.Equal(*data.Cur.UpdatedAt) {
logger.Info("image found, but image is identical, skipping")
@@ -167,16 +155,6 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error {
logger.Error("post could not update", "err", err.Error())
return fmt.Errorf("error for %s: %v", data.Filename, err)
}
-
- logger.Info(
- "found post tags, replacing with old tags",
- "tags", strings.Join(data.Tags, ","),
- )
- err = h.DBPool.ReplaceTagsForPost(data.Tags, data.Cur.ID)
- if err != nil {
- logger.Error("post could not replace tags", "err", err.Error())
- return fmt.Errorf("error for %s: %v", data.Filename, err)
- }
}
return nil
+0
-168
imgs/api.go
#
@@ -1,168 +0,0 @@
-package imgs
-
-import (
- "fmt"
- "html/template"
- "net/http"
- "net/url"
- "path/filepath"
-
- "github.com/picosh/pico/db"
- "github.com/picosh/pico/pgs"
- "github.com/picosh/pico/shared"
- "github.com/picosh/pico/shared/storage"
- "github.com/picosh/utils"
-)
-
-type PostPageData struct {
- ImgURL template.URL
-}
-
-type BlogPageData struct {
- Site *shared.SitePageData
- PageTitle string
- URL template.URL
- Username string
- Posts []template.URL
-}
-
-var Space = "imgs"
-
-func ImgsListHandler(w http.ResponseWriter, r *http.Request) {
- username := shared.GetUsernameFromRequest(r)
- dbpool := shared.GetDB(r)
- logger := shared.GetLogger(r)
- cfg := shared.GetCfg(r)
-
- user, err := dbpool.FindUserForName(username)
- if err != nil {
- logger.Info("blog not found", "username", username)
- http.Error(w, "blog not found", http.StatusNotFound)
- return
- }
-
- var posts []*db.Post
- pager := &db.Pager{Num: 1000, Page: 0}
- p, err := dbpool.FindPostsForUser(pager, user.ID, Space)
- posts = p.Data
-
- if err != nil {
- logger.Error(err.Error())
- http.Error(w, "could not fetch posts for blog", http.StatusInternalServerError)
- return
- }
-
- ts, err := shared.RenderTemplate(cfg, []string{
- cfg.StaticPath("html/imgs.page.tmpl"),
- })
-
- if err != nil {
- logger.Error(err.Error())
- http.Error(w, err.Error(), http.StatusInternalServerError)
- return
- }
-
- curl := shared.CreateURLFromRequest(cfg, r)
- postCollection := make([]template.URL, 0, len(posts))
- for _, post := range posts {
- url := cfg.ImgURL(curl, post.Username, post.Slug)
- postCollection = append(postCollection, template.URL(url))
- }
-
- data := BlogPageData{
- Site: cfg.GetSiteData(),
- PageTitle: fmt.Sprintf("%s imgs", username),
- URL: template.URL(cfg.FullBlogURL(curl, username)),
- Username: username,
- Posts: postCollection,
- }
-
- err = ts.Execute(w, data)
- if err != nil {
- logger.Error(err.Error())
- http.Error(w, err.Error(), http.StatusInternalServerError)
- }
-}
-
-func anyPerm(proj *db.Project) bool {
- return true
-}
-
-func ImgRequest(w http.ResponseWriter, r *http.Request) {
- subdomain := shared.GetSubdomain(r)
- cfg := shared.GetCfg(r)
- st := shared.GetStorage(r)
- dbpool := shared.GetDB(r)
- logger := shared.GetLogger(r)
- username := shared.GetUsernameFromRequest(r)
-
- user, err := dbpool.FindUserForName(username)
- if err != nil {
- logger.Info("user not found", "user", username)
- http.Error(w, "user not found", http.StatusNotFound)
- return
- }
-
- var imgOpts string
- var slug string
- if !cfg.IsSubdomains() || subdomain == "" {
- slug, _ = url.PathUnescape(shared.GetField(r, 1))
- imgOpts, _ = url.PathUnescape(shared.GetField(r, 2))
- } else {
- slug, _ = url.PathUnescape(shared.GetField(r, 0))
- imgOpts, _ = url.PathUnescape(shared.GetField(r, 1))
- }
-
- opts, err := storage.UriToImgProcessOpts(imgOpts)
- if err != nil {
- errMsg := fmt.Sprintf("error processing img options: %s", err.Error())
- logger.Info(errMsg)
- http.Error(w, errMsg, http.StatusUnprocessableEntity)
- return
- }
-
- // set default quality for web optimization
- if opts.Quality == 0 {
- opts.Quality = 80
- }
-
- ext := filepath.Ext(slug)
- // set default format to be webp
- if opts.Ext == "" && ext == "" {
- opts.Ext = "webp"
- }
-
- // Files can contain periods. `filepath.Ext` is greedy and will clip the last period in the slug
- // and call that a file extension so we want to be explicit about what
- // file extensions we clip here
- for _, fext := range cfg.AllowedExt {
- if ext == fext {
- // users might add the file extension when requesting an image
- // but we want to remove that
- slug = utils.SanitizeFileExt(slug)
- break
- }
- }
-
- post, err := FindImgPost(r, user, slug)
- if err != nil {
- errMsg := fmt.Sprintf("image not found %s/%s", user.Name, slug)
- logger.Info(errMsg)
- http.Error(w, errMsg, http.StatusNotFound)
- return
- }
-
- fname := post.Filename
- router := pgs.NewWebRouter(
- cfg,
- logger,
- dbpool,
- st,
- )
- router.ServeAsset(fname, opts, true, anyPerm, w, r)
-}
-
-func FindImgPost(r *http.Request, user *db.User, slug string) (*db.Post, error) {
- dbpool := shared.GetDB(r)
- return dbpool.FindPostWithSlug(slug, user.ID, Space)
-}
+0
-1
imgs/html/rss.page.tmpl
#
@@ -1,1 +0,0 @@
-<img src="{{.ImgURL}}" />
+0
-0
imgs/public/.gitkeep
#
+21
-12
prose/api.go
#
@@ -5,6 +5,7 @@ import (
"fmt"
"html/template"
"net/http"
+ "net/http/httputil"
"net/url"
"os"
"strconv"
@@ -16,7 +17,6 @@ import (
"github.com/gorilla/feeds"
"github.com/picosh/pico/db"
"github.com/picosh/pico/db/postgres"
- "github.com/picosh/pico/imgs"
"github.com/picosh/pico/shared"
"github.com/picosh/pico/shared/storage"
"github.com/picosh/utils"
@@ -438,14 +438,6 @@ func postHandler(w http.ResponseWriter, r *http.Request) {
WithStyles: withStyles,
}
} else {
- // TODO: HACK to support imgs slugs inside prose
- // We definitely want to kill this feature in time
- imgPost, err := imgs.FindImgPost(r, user, slug)
- if err == nil && imgPost != nil {
- imgs.ImgRequest(w, r)
- return
- }
-
notFound, err := dbpool.FindPostWithFilename("_404.md", user.ID, cfg.Space)
contents := template.HTML("Oops! we can't seem to find this post.")
title := "Post not found"
@@ -859,6 +851,24 @@ func createMainRoutes(staticRoutes []shared.Route) []shared.Route {
return routes
}
+func imgRequest(w http.ResponseWriter, r *http.Request) {
+ username := shared.GetUsernameFromRequest(r)
+ destUrl, err := url.Parse(fmt.Sprintf("https://%s-prose.pgs.sh", username))
+ if err != nil {
+ http.Error(w, "site not found", http.StatusNotFound)
+ return
+ }
+
+ proxy := httputil.NewSingleHostReverseProxy(destUrl)
+ oldDirector := proxy.Director
+ proxy.Director = func(r *http.Request) {
+ oldDirector(r)
+ r.Host = destUrl.Host
+ r.URL = destUrl
+ }
+ proxy.ServeHTTP(w, r)
+}
+
func createSubdomainRoutes(staticRoutes []shared.Route) []shared.Route {
routes := []shared.Route{
shared.NewRoute("GET", "/", blogHandler),
@@ -881,9 +891,8 @@ func createSubdomainRoutes(staticRoutes []shared.Route) []shared.Route {
routes = append(
routes,
shared.NewRoute("GET", "/raw/(.+)", postRawHandler),
- shared.NewRoute("GET", "/([^/]+)/(.+)", imgs.ImgRequest),
- shared.NewRoute("GET", "/(.+.(?:jpg|jpeg|png|gif|webp|svg))$", imgs.ImgRequest),
- shared.NewRoute("GET", "/i", imgs.ImgsListHandler),
+ shared.NewRoute("GET", "/([^/]+)/(.+)", imgRequest),
+ shared.NewRoute("GET", "/(.+.(?:jpg|jpeg|png|gif|webp|svg))$", imgRequest),
shared.NewRoute("GET", "/(.+)", postHandler),
)
+3
-0
prose/ssh.go
#
@@ -79,6 +79,9 @@ func StartSshServer() {
return
}
+ ctx := context.Background()
+ defer ctx.Done()
+
fileMap := map[string]filehandlers.ReadWriteHandler{
".md": filehandlers.NewScpPostHandler(dbh, cfg, hooks),
".css": filehandlers.NewScpPostHandler(dbh, cfg, hooks),