Patchset ps-94
chore(prose): migrate images to pgs
Eric Bower
filehandlers/imgs/img.go
+2
-24
prose/api.go
+21
-12
prose/ssh.go
+3
-0
chore(prose): migrate images to pgs
cmd/scripts/prose-imgs-migrate/main.go
link
+82
-0
+82
-0
1diff --git a/cmd/scripts/prose-imgs-migrate/main.go b/cmd/scripts/prose-imgs-migrate/main.go
2new file mode 100644
3index 0000000..0690dc4
4--- /dev/null
5+++ b/cmd/scripts/prose-imgs-migrate/main.go
6@@ -0,0 +1,82 @@
7+package main
8+
9+import (
10+ "bytes"
11+ "io"
12+ "log/slog"
13+ "path/filepath"
14+ "time"
15+
16+ "github.com/picosh/pico/db"
17+ "github.com/picosh/pico/db/postgres"
18+ "github.com/picosh/pico/prose"
19+ "github.com/picosh/pico/shared"
20+ "github.com/picosh/pico/shared/storage"
21+ sst "github.com/picosh/pobj/storage"
22+ sendUtils "github.com/picosh/send/utils"
23+)
24+
25+func bail(err error) {
26+ if err != nil {
27+ panic(err)
28+ }
29+}
30+
31+func upload(logger *slog.Logger, st storage.StorageServe, bucket sst.Bucket, fpath string, rdr io.Reader) error {
32+ toSite := filepath.Join("prose", fpath)
33+ logger.Info("uploading object", "bucket", bucket.Name, "object", toSite)
34+ buf := &bytes.Buffer{}
35+ size, err := io.Copy(buf, rdr)
36+ if err != nil {
37+ return err
38+ }
39+
40+ _, _, err = st.PutObject(bucket, toSite, buf, &sendUtils.FileEntry{
41+ Mtime: time.Now().Unix(),
42+ Size: size,
43+ })
44+ return err
45+}
46+
47+func images(logger *slog.Logger, st storage.StorageServe, bucket sst.Bucket, user *db.User) error {
48+ imgBucket, err := st.GetBucket(shared.GetImgsBucketName(user.ID))
49+ if err != nil {
50+ logger.Info("user does not have an images dir, skipping")
51+ return nil
52+ }
53+ imgs, err := st.ListObjects(imgBucket, "/", false)
54+ if err != nil {
55+ return err
56+ }
57+
58+ for _, inf := range imgs {
59+ rdr, _, err := st.GetObject(imgBucket, inf.Name())
60+ if err != nil {
61+ return err
62+ }
63+ err = upload(logger, st, bucket, inf.Name(), rdr)
64+ if err != nil {
65+ return err
66+ }
67+ }
68+
69+ return nil
70+}
71+
72+func main() {
73+ cfg := prose.NewConfigSite()
74+ logger := cfg.Logger
75+ picoDb := postgres.NewDB(cfg.DbURL, logger)
76+ st, err := storage.NewStorageMinio(logger, cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
77+ bail(err)
78+
79+ users, err := picoDb.FindUsers()
80+ bail(err)
81+
82+ for _, user := range users {
83+ bucket, err := st.UpsertBucket(shared.GetAssetBucketName(user.ID))
84+ bail(err)
85+ _, _ = picoDb.InsertProject(user.ID, "prose", "prose")
86+ bail(images(logger, st, bucket, user))
87+ }
88+}
filehandlers/imgs/handler.go
link
+8
-4
+8
-4
1diff --git a/filehandlers/imgs/handler.go b/filehandlers/imgs/handler.go
2index 3964090..5324009 100644
3--- a/filehandlers/imgs/handler.go
4+++ b/filehandlers/imgs/handler.go
5@@ -47,6 +47,10 @@ func NewUploadImgHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage.S
6 }
7 }
8
9+func (h *UploadImgHandler) getObjectPath(fpath string) string {
10+ return filepath.Join("prose", fpath)
11+}
12+
13 func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.FileInfo, sendutils.ReaderAtCloser, error) {
14 user, err := h.DBPool.FindUser(s.Permissions().Extensions["user_id"])
15 if err != nil {
16@@ -71,12 +75,12 @@ func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.F
17 FModTime: *post.UpdatedAt,
18 }
19
20- bucket, err := h.Storage.GetBucket(user.ID)
21+ bucket, err := h.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
22 if err != nil {
23 return nil, nil, err
24 }
25
26- contents, _, err := h.Storage.GetObject(bucket, post.Filename)
27+ contents, _, err := h.Storage.GetObject(bucket, h.getObjectPath(post.Filename))
28 if err != nil {
29 return nil, nil, err
30 }
31@@ -218,13 +222,13 @@ func (h *UploadImgHandler) Delete(s ssh.Session, entry *sendutils.FileEntry) err
32 return fmt.Errorf("error for %s: %v", filename, err)
33 }
34
35- bucket, err := h.Storage.UpsertBucket(user.ID)
36+ bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(user.ID))
37 if err != nil {
38 return err
39 }
40
41 logger.Info("deleting image")
42- err = h.Storage.DeleteObject(bucket, filename)
43+ err = h.Storage.DeleteObject(bucket, h.getObjectPath(filename))
44 if err != nil {
45 return err
46 }
filehandlers/imgs/img.go
link
+2
-24
+2
-24
1diff --git a/filehandlers/imgs/img.go b/filehandlers/imgs/img.go
2index 23cc0aa..83b296a 100644
3--- a/filehandlers/imgs/img.go
4+++ b/filehandlers/imgs/img.go
5@@ -49,7 +49,7 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
6 return nil
7 }
8
9- bucket, err := h.Storage.UpsertBucket(data.User.ID)
10+ bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(data.User.ID))
11 if err != nil {
12 return err
13 }
14@@ -58,7 +58,7 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
15
16 fname, _, err := h.Storage.PutObject(
17 bucket,
18- data.Filename,
19+ h.getObjectPath(data.Filename),
20 sendutils.NopReaderAtCloser(reader),
21 &sendutils.FileEntry{},
22 )
23@@ -128,18 +128,6 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error {
24 logger.Error("post could not create", "err", err.Error())
25 return fmt.Errorf("error for %s: %v", data.Filename, err)
26 }
27-
28- if len(data.Tags) > 0 {
29- logger.Info(
30- "found post tags, replacing with old tags",
31- "tags", strings.Join(data.Tags, ","),
32- )
33- err = h.DBPool.ReplaceTagsForPost(data.Tags, data.Post.ID)
34- if err != nil {
35- logger.Error("post could not replace tags", "err", err.Error())
36- return fmt.Errorf("error for %s: %v", data.Filename, err)
37- }
38- }
39 } else {
40 if data.Shasum == data.Cur.Shasum && modTime.Equal(*data.Cur.UpdatedAt) {
41 logger.Info("image found, but image is identical, skipping")
42@@ -167,16 +155,6 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error {
43 logger.Error("post could not update", "err", err.Error())
44 return fmt.Errorf("error for %s: %v", data.Filename, err)
45 }
46-
47- logger.Info(
48- "found post tags, replacing with old tags",
49- "tags", strings.Join(data.Tags, ","),
50- )
51- err = h.DBPool.ReplaceTagsForPost(data.Tags, data.Cur.ID)
52- if err != nil {
53- logger.Error("post could not replace tags", "err", err.Error())
54- return fmt.Errorf("error for %s: %v", data.Filename, err)
55- }
56 }
57
58 return nil
link
+0
-168
+0
-168
1diff --git a/imgs/api.go b/imgs/api.go
2deleted file mode 100644
3index 0d1ab10..0000000
4--- a/imgs/api.go
5+++ /dev/null
6@@ -1,168 +0,0 @@
7-package imgs
8-
9-import (
10- "fmt"
11- "html/template"
12- "net/http"
13- "net/url"
14- "path/filepath"
15-
16- "github.com/picosh/pico/db"
17- "github.com/picosh/pico/pgs"
18- "github.com/picosh/pico/shared"
19- "github.com/picosh/pico/shared/storage"
20- "github.com/picosh/utils"
21-)
22-
23-type PostPageData struct {
24- ImgURL template.URL
25-}
26-
27-type BlogPageData struct {
28- Site *shared.SitePageData
29- PageTitle string
30- URL template.URL
31- Username string
32- Posts []template.URL
33-}
34-
35-var Space = "imgs"
36-
37-func ImgsListHandler(w http.ResponseWriter, r *http.Request) {
38- username := shared.GetUsernameFromRequest(r)
39- dbpool := shared.GetDB(r)
40- logger := shared.GetLogger(r)
41- cfg := shared.GetCfg(r)
42-
43- user, err := dbpool.FindUserForName(username)
44- if err != nil {
45- logger.Info("blog not found", "username", username)
46- http.Error(w, "blog not found", http.StatusNotFound)
47- return
48- }
49-
50- var posts []*db.Post
51- pager := &db.Pager{Num: 1000, Page: 0}
52- p, err := dbpool.FindPostsForUser(pager, user.ID, Space)
53- posts = p.Data
54-
55- if err != nil {
56- logger.Error(err.Error())
57- http.Error(w, "could not fetch posts for blog", http.StatusInternalServerError)
58- return
59- }
60-
61- ts, err := shared.RenderTemplate(cfg, []string{
62- cfg.StaticPath("html/imgs.page.tmpl"),
63- })
64-
65- if err != nil {
66- logger.Error(err.Error())
67- http.Error(w, err.Error(), http.StatusInternalServerError)
68- return
69- }
70-
71- curl := shared.CreateURLFromRequest(cfg, r)
72- postCollection := make([]template.URL, 0, len(posts))
73- for _, post := range posts {
74- url := cfg.ImgURL(curl, post.Username, post.Slug)
75- postCollection = append(postCollection, template.URL(url))
76- }
77-
78- data := BlogPageData{
79- Site: cfg.GetSiteData(),
80- PageTitle: fmt.Sprintf("%s imgs", username),
81- URL: template.URL(cfg.FullBlogURL(curl, username)),
82- Username: username,
83- Posts: postCollection,
84- }
85-
86- err = ts.Execute(w, data)
87- if err != nil {
88- logger.Error(err.Error())
89- http.Error(w, err.Error(), http.StatusInternalServerError)
90- }
91-}
92-
93-func anyPerm(proj *db.Project) bool {
94- return true
95-}
96-
97-func ImgRequest(w http.ResponseWriter, r *http.Request) {
98- subdomain := shared.GetSubdomain(r)
99- cfg := shared.GetCfg(r)
100- st := shared.GetStorage(r)
101- dbpool := shared.GetDB(r)
102- logger := shared.GetLogger(r)
103- username := shared.GetUsernameFromRequest(r)
104-
105- user, err := dbpool.FindUserForName(username)
106- if err != nil {
107- logger.Info("user not found", "user", username)
108- http.Error(w, "user not found", http.StatusNotFound)
109- return
110- }
111-
112- var imgOpts string
113- var slug string
114- if !cfg.IsSubdomains() || subdomain == "" {
115- slug, _ = url.PathUnescape(shared.GetField(r, 1))
116- imgOpts, _ = url.PathUnescape(shared.GetField(r, 2))
117- } else {
118- slug, _ = url.PathUnescape(shared.GetField(r, 0))
119- imgOpts, _ = url.PathUnescape(shared.GetField(r, 1))
120- }
121-
122- opts, err := storage.UriToImgProcessOpts(imgOpts)
123- if err != nil {
124- errMsg := fmt.Sprintf("error processing img options: %s", err.Error())
125- logger.Info(errMsg)
126- http.Error(w, errMsg, http.StatusUnprocessableEntity)
127- return
128- }
129-
130- // set default quality for web optimization
131- if opts.Quality == 0 {
132- opts.Quality = 80
133- }
134-
135- ext := filepath.Ext(slug)
136- // set default format to be webp
137- if opts.Ext == "" && ext == "" {
138- opts.Ext = "webp"
139- }
140-
141- // Files can contain periods. `filepath.Ext` is greedy and will clip the last period in the slug
142- // and call that a file extension so we want to be explicit about what
143- // file extensions we clip here
144- for _, fext := range cfg.AllowedExt {
145- if ext == fext {
146- // users might add the file extension when requesting an image
147- // but we want to remove that
148- slug = utils.SanitizeFileExt(slug)
149- break
150- }
151- }
152-
153- post, err := FindImgPost(r, user, slug)
154- if err != nil {
155- errMsg := fmt.Sprintf("image not found %s/%s", user.Name, slug)
156- logger.Info(errMsg)
157- http.Error(w, errMsg, http.StatusNotFound)
158- return
159- }
160-
161- fname := post.Filename
162- router := pgs.NewWebRouter(
163- cfg,
164- logger,
165- dbpool,
166- st,
167- )
168- router.ServeAsset(fname, opts, true, anyPerm, w, r)
169-}
170-
171-func FindImgPost(r *http.Request, user *db.User, slug string) (*db.Post, error) {
172- dbpool := shared.GetDB(r)
173- return dbpool.FindPostWithSlug(slug, user.ID, Space)
174-}
link
+0
-1
+0
-1
link
+0
-0
+0
-0
prose/api.go
link
+21
-12
+21
-12
1diff --git a/prose/api.go b/prose/api.go
2index 8c867ae..7760436 100644
3--- a/prose/api.go
4+++ b/prose/api.go
5@@ -5,6 +5,7 @@ import (
6 "fmt"
7 "html/template"
8 "net/http"
9+ "net/http/httputil"
10 "net/url"
11 "os"
12 "strconv"
13@@ -16,7 +17,6 @@ import (
14 "github.com/gorilla/feeds"
15 "github.com/picosh/pico/db"
16 "github.com/picosh/pico/db/postgres"
17- "github.com/picosh/pico/imgs"
18 "github.com/picosh/pico/shared"
19 "github.com/picosh/pico/shared/storage"
20 "github.com/picosh/utils"
21@@ -438,14 +438,6 @@ func postHandler(w http.ResponseWriter, r *http.Request) {
22 WithStyles: withStyles,
23 }
24 } else {
25- // TODO: HACK to support imgs slugs inside prose
26- // We definitely want to kill this feature in time
27- imgPost, err := imgs.FindImgPost(r, user, slug)
28- if err == nil && imgPost != nil {
29- imgs.ImgRequest(w, r)
30- return
31- }
32-
33 notFound, err := dbpool.FindPostWithFilename("_404.md", user.ID, cfg.Space)
34 contents := template.HTML("Oops! we can't seem to find this post.")
35 title := "Post not found"
36@@ -859,6 +851,24 @@ func createMainRoutes(staticRoutes []shared.Route) []shared.Route {
37 return routes
38 }
39
40+func imgRequest(w http.ResponseWriter, r *http.Request) {
41+ username := shared.GetUsernameFromRequest(r)
42+ destUrl, err := url.Parse(fmt.Sprintf("https://%s-prose.pgs.sh", username))
43+ if err != nil {
44+ http.Error(w, "site not found", http.StatusNotFound)
45+ return
46+ }
47+
48+ proxy := httputil.NewSingleHostReverseProxy(destUrl)
49+ oldDirector := proxy.Director
50+ proxy.Director = func(r *http.Request) {
51+ oldDirector(r)
52+ r.Host = destUrl.Host
53+ r.URL = destUrl
54+ }
55+ proxy.ServeHTTP(w, r)
56+}
57+
58 func createSubdomainRoutes(staticRoutes []shared.Route) []shared.Route {
59 routes := []shared.Route{
60 shared.NewRoute("GET", "/", blogHandler),
61@@ -881,9 +891,8 @@ func createSubdomainRoutes(staticRoutes []shared.Route) []shared.Route {
62 routes = append(
63 routes,
64 shared.NewRoute("GET", "/raw/(.+)", postRawHandler),
65- shared.NewRoute("GET", "/([^/]+)/(.+)", imgs.ImgRequest),
66- shared.NewRoute("GET", "/(.+.(?:jpg|jpeg|png|gif|webp|svg))$", imgs.ImgRequest),
67- shared.NewRoute("GET", "/i", imgs.ImgsListHandler),
68+ shared.NewRoute("GET", "/([^/]+)/(.+)", imgRequest),
69+ shared.NewRoute("GET", "/(.+.(?:jpg|jpeg|png|gif|webp|svg))$", imgRequest),
70 shared.NewRoute("GET", "/(.+)", postHandler),
71 )
72
prose/ssh.go
link
+3
-0
+3
-0
1diff --git a/prose/ssh.go b/prose/ssh.go
2index 547bffa..427656d 100644
3--- a/prose/ssh.go
4+++ b/prose/ssh.go
5@@ -79,6 +79,9 @@ func StartSshServer() {
6 return
7 }
8
9+ ctx := context.Background()
10+ defer ctx.Done()
11+
12 fileMap := map[string]filehandlers.ReadWriteHandler{
13 ".md": filehandlers.NewScpPostHandler(dbh, cfg, hooks),
14 ".css": filehandlers.NewScpPostHandler(dbh, cfg, hooks),