pico

chore(prose): migrate images to pgs open rss
created pr with 44.1 on 2025-01-18T20:18:40Z · by c8ef7d19
added 44.2 on 2025-01-18T20:20:30Z · by c8ef7d19
1: 89e0a31 ! 1: aa126f8 chore(prose): migrate images to pgs
changed status to accepted on 2025-02-01T18:43:36Z · by c8ef7d19
cmds
checkout latest patchset: 
ssh pr.pico.sh print 44 | git am -3
checkout any patchset in a patch request: 
ssh pr.pico.sh print 44.[rev] | git am -3
add changes to patch request: 
git format-patch main --stdout | ssh pr.pico.sh pr add 44
set PR to open (enables RSS notifications): 
ssh pr.pico.sh pr open 44
set PR to draft (stops RSS notifications): 
ssh pr.pico.sh pr draft 44

Patchset 44.2 on 2025-01-18T20:20:30Z · commit aa126f8

→ chore(prose): migrate images to pgs
Eric Bower 2025-01-18T14:27:41Z
+82 -0 cmd/scripts/prose-imgs-migrate/main.go #
......@@ -0,0 +1,82 @@
1+package main
2+
3+import (
4+ "bytes"
5+ "io"
6+ "log/slog"
7+ "path/filepath"
8+ "time"
9+
10+ "github.com/picosh/pico/db"
11+ "github.com/picosh/pico/db/postgres"
12+ "github.com/picosh/pico/prose"
13+ "github.com/picosh/pico/shared"
14+ "github.com/picosh/pico/shared/storage"
15+ sst "github.com/picosh/pobj/storage"
16+ sendUtils "github.com/picosh/send/utils"
17+)
18+
19+func bail(err error) {
20+ if err != nil {
21+ panic(err)
22+ }
23+}
24+
25+func upload(logger *slog.Logger, st storage.StorageServe, bucket sst.Bucket, fpath string, rdr io.Reader) error {
26+ toSite := filepath.Join("prose", fpath)
27+ logger.Info("uploading object", "bucket", bucket.Name, "object", toSite)
28+ buf := &bytes.Buffer{}
29+ size, err := io.Copy(buf, rdr)
30+ if err != nil {
31+ return err
32+ }
33+
34+ _, _, err = st.PutObject(bucket, toSite, buf, &sendUtils.FileEntry{
35+ Mtime: time.Now().Unix(),
36+ Size: size,
37+ })
38+ return err
39+}
40+
41+func images(logger *slog.Logger, st storage.StorageServe, bucket sst.Bucket, user *db.User) error {
42+ imgBucket, err := st.GetBucket(shared.GetImgsBucketName(user.ID))
43+ if err != nil {
44+ logger.Info("user does not have an images dir, skipping")
45+ return nil
46+ }
47+ imgs, err := st.ListObjects(imgBucket, "/", false)
48+ if err != nil {
49+ return err
50+ }
51+
52+ for _, inf := range imgs {
53+ rdr, _, err := st.GetObject(imgBucket, inf.Name())
54+ if err != nil {
55+ return err
56+ }
57+ err = upload(logger, st, bucket, inf.Name(), rdr)
58+ if err != nil {
59+ return err
60+ }
61+ }
62+
63+ return nil
64+}
65+
66+func main() {
67+ cfg := prose.NewConfigSite()
68+ logger := cfg.Logger
69+ picoDb := postgres.NewDB(cfg.DbURL, logger)
70+ st, err := storage.NewStorageMinio(logger, cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
71+ bail(err)
72+
73+ users, err := picoDb.FindUsers()
74+ bail(err)
75+
76+ for _, user := range users {
77+ bucket, err := st.UpsertBucket(shared.GetAssetBucketName(user.ID))
78+ bail(err)
79+ _, _ = picoDb.InsertProject(user.ID, "prose", "prose")
80+ bail(images(logger, st, bucket, user))
81+ }
82+}
+8 -4 filehandlers/imgs/handler.go #
......@@ -47,6 +47,10 @@ func NewUploadImgHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage.S
4747 }
4848 }
4949
50+func (h *UploadImgHandler) getObjectPath(fpath string) string {
51+ return filepath.Join("prose", fpath)
52+}
53+
5054 func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.FileInfo, sendutils.ReaderAtCloser, error) {
5155 user, err := h.DBPool.FindUser(s.Permissions().Extensions["user_id"])
5256 if err != nil {
......@@ -71,12 +75,12 @@ func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.F
7175 FModTime: *post.UpdatedAt,
7276 }
7377
74- bucket, err := h.Storage.GetBucket(user.ID)
78+ bucket, err := h.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
7579 if err != nil {
7680 return nil, nil, err
7781 }
7882
79- contents, _, err := h.Storage.GetObject(bucket, post.Filename)
83+ contents, _, err := h.Storage.GetObject(bucket, h.getObjectPath(post.Filename))
8084 if err != nil {
8185 return nil, nil, err
8286 }
......@@ -218,13 +222,13 @@ func (h *UploadImgHandler) Delete(s ssh.Session, entry *sendutils.FileEntry) err
218222 return fmt.Errorf("error for %s: %v", filename, err)
219223 }
220224
221- bucket, err := h.Storage.UpsertBucket(user.ID)
225+ bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(user.ID))
222226 if err != nil {
223227 return err
224228 }
225229
226230 logger.Info("deleting image")
227- err = h.Storage.DeleteObject(bucket, filename)
231+ err = h.Storage.DeleteObject(bucket, h.getObjectPath(filename))
228232 if err != nil {
229233 return err
230234 }
+2 -24 filehandlers/imgs/img.go #
......@@ -49,7 +49,7 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
4949 return nil
5050 }
5151
52- bucket, err := h.Storage.UpsertBucket(data.User.ID)
52+ bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(data.User.ID))
5353 if err != nil {
5454 return err
5555 }
......@@ -58,7 +58,7 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
5858
5959 fname, _, err := h.Storage.PutObject(
6060 bucket,
61- data.Filename,
61+ h.getObjectPath(data.Filename),
6262 sendutils.NopReaderAtCloser(reader),
6363 &sendutils.FileEntry{},
6464 )
......@@ -128,18 +128,6 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error {
128128 logger.Error("post could not create", "err", err.Error())
129129 return fmt.Errorf("error for %s: %v", data.Filename, err)
130130 }
131-
132- if len(data.Tags) > 0 {
133- logger.Info(
134- "found post tags, replacing with old tags",
135- "tags", strings.Join(data.Tags, ","),
136- )
137- err = h.DBPool.ReplaceTagsForPost(data.Tags, data.Post.ID)
138- if err != nil {
139- logger.Error("post could not replace tags", "err", err.Error())
140- return fmt.Errorf("error for %s: %v", data.Filename, err)
141- }
142- }
143131 } else {
144132 if data.Shasum == data.Cur.Shasum && modTime.Equal(*data.Cur.UpdatedAt) {
145133 logger.Info("image found, but image is identical, skipping")
......@@ -167,16 +155,6 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error {
167155 logger.Error("post could not update", "err", err.Error())
168156 return fmt.Errorf("error for %s: %v", data.Filename, err)
169157 }
170-
171- logger.Info(
172- "found post tags, replacing with old tags",
173- "tags", strings.Join(data.Tags, ","),
174- )
175- err = h.DBPool.ReplaceTagsForPost(data.Tags, data.Cur.ID)
176- if err != nil {
177- logger.Error("post could not replace tags", "err", err.Error())
178- return fmt.Errorf("error for %s: %v", data.Filename, err)
179- }
180158 }
181159
182160 return nil
+0 -168 imgs/api.go #
......@@ -1,168 +0,0 @@
1-package imgs
2-
3-import (
4- "fmt"
5- "html/template"
6- "net/http"
7- "net/url"
8- "path/filepath"
9-
10- "github.com/picosh/pico/db"
11- "github.com/picosh/pico/pgs"
12- "github.com/picosh/pico/shared"
13- "github.com/picosh/pico/shared/storage"
14- "github.com/picosh/utils"
15-)
16-
17-type PostPageData struct {
18- ImgURL template.URL
19-}
20-
21-type BlogPageData struct {
22- Site *shared.SitePageData
23- PageTitle string
24- URL template.URL
25- Username string
26- Posts []template.URL
27-}
28-
29-var Space = "imgs"
30-
31-func ImgsListHandler(w http.ResponseWriter, r *http.Request) {
32- username := shared.GetUsernameFromRequest(r)
33- dbpool := shared.GetDB(r)
34- logger := shared.GetLogger(r)
35- cfg := shared.GetCfg(r)
36-
37- user, err := dbpool.FindUserForName(username)
38- if err != nil {
39- logger.Info("blog not found", "username", username)
40- http.Error(w, "blog not found", http.StatusNotFound)
41- return
42- }
43-
44- var posts []*db.Post
45- pager := &db.Pager{Num: 1000, Page: 0}
46- p, err := dbpool.FindPostsForUser(pager, user.ID, Space)
47- posts = p.Data
48-
49- if err != nil {
50- logger.Error(err.Error())
51- http.Error(w, "could not fetch posts for blog", http.StatusInternalServerError)
52- return
53- }
54-
55- ts, err := shared.RenderTemplate(cfg, []string{
56- cfg.StaticPath("html/imgs.page.tmpl"),
57- })
58-
59- if err != nil {
60- logger.Error(err.Error())
61- http.Error(w, err.Error(), http.StatusInternalServerError)
62- return
63- }
64-
65- curl := shared.CreateURLFromRequest(cfg, r)
66- postCollection := make([]template.URL, 0, len(posts))
67- for _, post := range posts {
68- url := cfg.ImgURL(curl, post.Username, post.Slug)
69- postCollection = append(postCollection, template.URL(url))
70- }
71-
72- data := BlogPageData{
73- Site: cfg.GetSiteData(),
74- PageTitle: fmt.Sprintf("%s imgs", username),
75- URL: template.URL(cfg.FullBlogURL(curl, username)),
76- Username: username,
77- Posts: postCollection,
78- }
79-
80- err = ts.Execute(w, data)
81- if err != nil {
82- logger.Error(err.Error())
83- http.Error(w, err.Error(), http.StatusInternalServerError)
84- }
85-}
86-
87-func anyPerm(proj *db.Project) bool {
88- return true
89-}
90-
91-func ImgRequest(w http.ResponseWriter, r *http.Request) {
92- subdomain := shared.GetSubdomain(r)
93- cfg := shared.GetCfg(r)
94- st := shared.GetStorage(r)
95- dbpool := shared.GetDB(r)
96- logger := shared.GetLogger(r)
97- username := shared.GetUsernameFromRequest(r)
98-
99- user, err := dbpool.FindUserForName(username)
100- if err != nil {
101- logger.Info("user not found", "user", username)
102- http.Error(w, "user not found", http.StatusNotFound)
103- return
104- }
105-
106- var imgOpts string
107- var slug string
108- if !cfg.IsSubdomains() || subdomain == "" {
109- slug, _ = url.PathUnescape(shared.GetField(r, 1))
110- imgOpts, _ = url.PathUnescape(shared.GetField(r, 2))
111- } else {
112- slug, _ = url.PathUnescape(shared.GetField(r, 0))
113- imgOpts, _ = url.PathUnescape(shared.GetField(r, 1))
114- }
115-
116- opts, err := storage.UriToImgProcessOpts(imgOpts)
117- if err != nil {
118- errMsg := fmt.Sprintf("error processing img options: %s", err.Error())
119- logger.Info(errMsg)
120- http.Error(w, errMsg, http.StatusUnprocessableEntity)
121- return
122- }
123-
124- // set default quality for web optimization
125- if opts.Quality == 0 {
126- opts.Quality = 80
127- }
128-
129- ext := filepath.Ext(slug)
130- // set default format to be webp
131- if opts.Ext == "" && ext == "" {
132- opts.Ext = "webp"
133- }
134-
135- // Files can contain periods. `filepath.Ext` is greedy and will clip the last period in the slug
136- // and call that a file extension so we want to be explicit about what
137- // file extensions we clip here
138- for _, fext := range cfg.AllowedExt {
139- if ext == fext {
140- // users might add the file extension when requesting an image
141- // but we want to remove that
142- slug = utils.SanitizeFileExt(slug)
143- break
144- }
145- }
146-
147- post, err := FindImgPost(r, user, slug)
148- if err != nil {
149- errMsg := fmt.Sprintf("image not found %s/%s", user.Name, slug)
150- logger.Info(errMsg)
151- http.Error(w, errMsg, http.StatusNotFound)
152- return
153- }
154-
155- fname := post.Filename
156- router := pgs.NewWebRouter(
157- cfg,
158- logger,
159- dbpool,
160- st,
161- )
162- router.ServeAsset(fname, opts, true, anyPerm, w, r)
163-}
164-
165-func FindImgPost(r *http.Request, user *db.User, slug string) (*db.Post, error) {
166- dbpool := shared.GetDB(r)
167- return dbpool.FindPostWithSlug(slug, user.ID, Space)
168-}
+0 -1 imgs/html/rss.page.tmpl #
......@@ -1,1 +0,0 @@
1-<img src="{{.ImgURL}}" />
+0 -0 imgs/public/.gitkeep #
+21 -12 prose/api.go #
......@@ -5,6 +5,7 @@ import (
55 "fmt"
66 "html/template"
77 "net/http"
8+ "net/http/httputil"
89 "net/url"
910 "os"
1011 "strconv"
......@@ -16,7 +17,6 @@ import (
1617 "github.com/gorilla/feeds"
1718 "github.com/picosh/pico/db"
1819 "github.com/picosh/pico/db/postgres"
19- "github.com/picosh/pico/imgs"
2020 "github.com/picosh/pico/shared"
2121 "github.com/picosh/pico/shared/storage"
2222 "github.com/picosh/utils"
......@@ -438,14 +438,6 @@ func postHandler(w http.ResponseWriter, r *http.Request) {
438438 WithStyles: withStyles,
439439 }
440440 } else {
441- // TODO: HACK to support imgs slugs inside prose
442- // We definitely want to kill this feature in time
443- imgPost, err := imgs.FindImgPost(r, user, slug)
444- if err == nil && imgPost != nil {
445- imgs.ImgRequest(w, r)
446- return
447- }
448-
449441 notFound, err := dbpool.FindPostWithFilename("_404.md", user.ID, cfg.Space)
450442 contents := template.HTML("Oops! we can't seem to find this post.")
451443 title := "Post not found"
......@@ -859,6 +851,24 @@ func createMainRoutes(staticRoutes []shared.Route) []shared.Route {
859851 return routes
860852 }
861853
854+func imgRequest(w http.ResponseWriter, r *http.Request) {
855+ username := shared.GetUsernameFromRequest(r)
856+ destUrl, err := url.Parse(fmt.Sprintf("https://%s-prose.pgs.sh", username))
857+ if err != nil {
858+ http.Error(w, "site not found", http.StatusNotFound)
859+ return
860+ }
861+
862+ proxy := httputil.NewSingleHostReverseProxy(destUrl)
863+ oldDirector := proxy.Director
864+ proxy.Director = func(r *http.Request) {
865+ oldDirector(r)
866+ r.Host = destUrl.Host
867+ r.URL = destUrl
868+ }
869+ proxy.ServeHTTP(w, r)
870+}
871+
862872 func createSubdomainRoutes(staticRoutes []shared.Route) []shared.Route {
863873 routes := []shared.Route{
864874 shared.NewRoute("GET", "/", blogHandler),
......@@ -881,9 +891,8 @@ func createSubdomainRoutes(staticRoutes []shared.Route) []shared.Route {
881891 routes = append(
882892 routes,
883893 shared.NewRoute("GET", "/raw/(.+)", postRawHandler),
884- shared.NewRoute("GET", "/([^/]+)/(.+)", imgs.ImgRequest),
885- shared.NewRoute("GET", "/(.+.(?:jpg|jpeg|png|gif|webp|svg))$", imgs.ImgRequest),
886- shared.NewRoute("GET", "/i", imgs.ImgsListHandler),
894+ shared.NewRoute("GET", "/([^/]+)/(.+)", imgRequest),
895+ shared.NewRoute("GET", "/(.+.(?:jpg|jpeg|png|gif|webp|svg))$", imgRequest),
887896 shared.NewRoute("GET", "/(.+)", postHandler),
888897 )
889898
Back to top