pico

feat(auth): subscribe to pico's metric-drain pipe rss
created pr with 32.1 on 2024-11-12T21:13:35Z · by c8ef7d19
added 32.2 on 2024-11-14T15:42:32Z · by c8ef7d19
1: 7ec3569 = 1: 7ec3569 feat(auth): subscribe to pico's metric-drain pipe
2: 8a197f0 = 2: 8a197f0 chore: update pubsub
3: d4bda15 = 3: d4bda15 refactor: use pipe for analytics
-: ------- > 4: 2b7c358 chore: prep for release
cmds
checkout latest patchset: 
ssh pr.pico.sh print 32 | git am -3
checkout any patchset in a patch request: 
ssh pr.pico.sh print 32.[rev] | git am -3
add changes to patch request: 
git format-patch main --stdout | ssh pr.pico.sh pr add 32

Patchset 32.2 on 2024-11-14T15:42:32Z · commit 2b7c358

feat(auth): subscribe to pico's metric-drain pipe
Eric Bower 2024-11-11T02:41:27Z
 chore: update pubsub
Eric Bower 2024-11-12T16:29:38Z
 refactor: use pipe for analytics
Eric Bower 2024-11-12T21:12:05Z
 → chore: prep for release
Eric Bower 2024-11-14T15:10:28Z
+2 -1 Makefile #
......@@ -130,10 +130,11 @@ migrate:
130130 $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20240324_add_analytics_table.sql
131131 $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20240819_add_projects_blocked.sql
132132 $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241028_add_analytics_indexes.sql
133+ $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241114_add_namespace_to_analytics.sql
133134 .PHONY: migrate
134135
135136 latest:
136- $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241028_add_analytics_indexes.sql
137+ $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241114_add_namespace_to_analytics.sql
137138 .PHONY: latest
138139
139140 psql:
+16 -2 auth/api.go #
......@@ -5,6 +5,7 @@ import (
55 "context"
66 "crypto/hmac"
77 "encoding/json"
8+ "errors"
89 "fmt"
910 "html/template"
1011 "io"
......@@ -641,7 +642,7 @@ func handler(routes []shared.Route, client *Client) http.HandlerFunc {
641642 }
642643 }
643644
644-func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger) {
645+func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secret string) {
645646 conn := shared.NewPicoPipeClient()
646647 stdoutPipe, err := pubsub.RemoteSub("sub metric-drain -k", ctx, conn)
647648
......@@ -659,6 +660,14 @@ func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger) {
659660 logger.Error("json unmarshal", "err", err)
660661 continue
661662 }
663+
664+ err = shared.AnalyticsVisitFromVisit(&view, dbpool, secret)
665+ if err != nil {
666+ if !errors.Is(err, shared.ErrAnalyticsDisabled) {
667+ logger.Info("could not record analytics view", "reason", err)
668+ }
669+ }
670+
662671 err = dbpool.InsertVisit(&view)
663672 if err != nil {
664673 logger.Error("could not insert view record", "err", err)
......@@ -672,6 +681,7 @@ type AuthCfg struct {
672681 DbURL string
673682 Domain string
674683 Issuer string
684+ Secret string
675685 }
676686
677687 func StartApiServer() {
......@@ -682,6 +692,10 @@ func StartApiServer() {
682692 Issuer: utils.GetEnv("AUTH_ISSUER", "pico.sh"),
683693 Domain: utils.GetEnv("AUTH_DOMAIN", "http://0.0.0.0:3000"),
684694 Port: utils.GetEnv("AUTH_WEB_PORT", "3000"),
695+ Secret: utils.GetEnv("PICO_SECRET", ""),
696+ }
697+ if cfg.Secret == "" {
698+ panic("must provide PICO_SECRET environment variable")
685699 }
686700
687701 logger := shared.CreateLogger("auth")
......@@ -696,7 +710,7 @@ func StartApiServer() {
696710
697711 ctx := context.Background()
698712 // gather metrics in the auth service
699- go metricDrainSub(ctx, db, logger)
713+ go metricDrainSub(ctx, db, logger, cfg.Secret)
700714 defer ctx.Done()
701715
702716 routes := createMainRoutes()
+1 -0 db/db.go #
......@@ -165,6 +165,7 @@ type AnalyticsVisits struct {
165165 UserID string `json:"user_id"`
166166 ProjectID string `json:"project_id"`
167167 PostID string `json:"post_id"`
168+ Namespace string `json:"namespace"`
168169 Host string `json:"host"`
169170 Path string `json:"path"`
170171 IpAddress string `json:"ip_adress"`
+12 -11 db/postgres/storage.go #
......@@ -984,18 +984,19 @@ func newNullString(s string) sql.NullString {
984984 }
985985 }
986986
987-func (me *PsqlDB) InsertVisit(view *db.AnalyticsVisits) error {
987+func (me *PsqlDB) InsertVisit(visit *db.AnalyticsVisits) error {
988988 _, err := me.Db.Exec(
989- `INSERT INTO analytics_visits (user_id, project_id, post_id, host, path, ip_address, user_agent, referer, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9);`,
990- view.UserID,
991- newNullString(view.ProjectID),
992- newNullString(view.PostID),
993- view.Host,
994- view.Path,
995- view.IpAddress,
996- view.UserAgent,
997- view.Referer,
998- view.Status,
989+ `INSERT INTO analytics_visits (user_id, project_id, post_id, namespace, host, path, ip_address, user_agent, referer, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);`,
990+ visit.UserID,
991+ newNullString(visit.ProjectID),
992+ newNullString(visit.PostID),
993+ newNullString(visit.Namespace),
994+ visit.Host,
995+ visit.Path,
996+ visit.IpAddress,
997+ visit.UserAgent,
998+ visit.Referer,
999+ visit.Status,
9991000 )
10001001 return err
10011002 }
+1 -1 go.mod #
......@@ -36,7 +36,7 @@ require (
3636 github.com/muesli/termenv v0.15.3-0.20240912151726-82936c5ea257
3737 github.com/neurosnap/go-exif-remove v0.0.0-20221010134343-50d1e3c35577
3838 github.com/picosh/pobj v0.0.0-20241016194248-c39198b2ff23
39- github.com/picosh/pubsub v0.0.0-20241112151357-866d44c53659
39+ github.com/picosh/pubsub v0.0.0-20241114025640-35db438302b4
4040 github.com/picosh/send v0.0.0-20241107150437-0febb0049b4f
4141 github.com/picosh/tunkit v0.0.0-20240905223921-532404cef9d9
4242 github.com/picosh/utils v0.0.0-20241018143404-b351d5d765f3
+2 -2 go.sum #
......@@ -269,8 +269,8 @@ github.com/picosh/go-rsync-receiver v0.0.0-20240709135253-1daf4b12a9fc h1:bvcsoO
269269 github.com/picosh/go-rsync-receiver v0.0.0-20240709135253-1daf4b12a9fc/go.mod h1:i0iR3W4GSm1PuvVxB9OH32E5jP+CYkVb2NQSe0JCtlo=
270270 github.com/picosh/pobj v0.0.0-20241016194248-c39198b2ff23 h1:NEJ5a4UXeF0/X7xmYNzXcwLQID9DwgazlqkMMC5zZ3M=
271271 github.com/picosh/pobj v0.0.0-20241016194248-c39198b2ff23/go.mod h1:cF+eAl4G1vU+WOD8cYCKaxokHo6MWmbR8J4/SJnvESg=
272-github.com/picosh/pubsub v0.0.0-20241112151357-866d44c53659 h1:HmRi+QkAcKkOcLD90xbf7qZy95muQEd/DqttK9xtpHk=
273-github.com/picosh/pubsub v0.0.0-20241112151357-866d44c53659/go.mod h1:m6ZZpg+lZB3XTIKlbSqQgi4NrBPtARv23b8vGYDoCo4=
272+github.com/picosh/pubsub v0.0.0-20241114025640-35db438302b4 h1:pITSRXb9NDGdC6AmuS3JE+8Ek4/pUG7tXJPP3cOaqf4=
273+github.com/picosh/pubsub v0.0.0-20241114025640-35db438302b4/go.mod h1:m6ZZpg+lZB3XTIKlbSqQgi4NrBPtARv23b8vGYDoCo4=
274274 github.com/picosh/send v0.0.0-20241107150437-0febb0049b4f h1:pdEh1Z7zH5Og9nS7jRuqwup3bcPsC6faDNQ6mgrV9ws=
275275 github.com/picosh/send v0.0.0-20241107150437-0febb0049b4f/go.mod h1:RAgLDK3LrDK6pNeXtU9tjo28obl5DxShcTUk2nm/KCM=
276276 github.com/picosh/senpai v0.0.0-20240503200611-af89e73973b0 h1:pBRIbiCj7K6rGELijb//dYhyCo8A3fvxW5dijrJVtjs=
+0 -5 pgs/config.go #
......@@ -20,13 +20,8 @@ func NewConfigSite() *shared.ConfigSite {
2020 minioUser := utils.GetEnv("MINIO_ROOT_USER", "")
2121 minioPass := utils.GetEnv("MINIO_ROOT_PASSWORD", "")
2222 dbURL := utils.GetEnv("DATABASE_URL", "")
23- secret := utils.GetEnv("PICO_SECRET", "")
24- if secret == "" {
25- panic("must provide PICO_SECRET environment variable")
26- }
2723
2824 cfg := shared.ConfigSite{
29- Secret: secret,
3025 Domain: domain,
3126 Port: port,
3227 Protocol: protocol,
+2 -2 pgs/web_asset_handler.go #
......@@ -157,7 +157,7 @@ func (h *ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
157157 )
158158 // track 404s
159159 ch := h.AnalyticsQueue
160- view, err := shared.AnalyticsVisitFromRequest(r, h.Dbpool, h.UserID, h.Cfg.Secret)
160+ view, err := shared.AnalyticsVisitFromRequest(r, h.Dbpool, h.UserID)
161161 if err == nil {
162162 view.ProjectID = h.ProjectID
163163 view.Status = http.StatusNotFound
......@@ -236,7 +236,7 @@ func (h *ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
236236 if finContentType == "text/html" {
237237 // track visit
238238 ch := h.AnalyticsQueue
239- view, err := shared.AnalyticsVisitFromRequest(r, h.Dbpool, h.UserID, h.Cfg.Secret)
239+ view, err := shared.AnalyticsVisitFromRequest(r, h.Dbpool, h.UserID)
240240 if err == nil {
241241 view.ProjectID = h.ProjectID
242242 ch <- view
+2 -2 prose/api.go #
......@@ -272,7 +272,7 @@ func blogHandler(w http.ResponseWriter, r *http.Request) {
272272
273273 // track visit
274274 ch := shared.GetAnalyticsQueue(r)
275- view, err := shared.AnalyticsVisitFromRequest(r, dbpool, user.ID, cfg.Secret)
275+ view, err := shared.AnalyticsVisitFromRequest(r, dbpool, user.ID)
276276 if err == nil {
277277 ch <- view
278278 } else {
......@@ -426,7 +426,7 @@ func postHandler(w http.ResponseWriter, r *http.Request) {
426426 }
427427
428428 // track visit
429- view, err := shared.AnalyticsVisitFromRequest(r, dbpool, user.ID, cfg.Secret)
429+ view, err := shared.AnalyticsVisitFromRequest(r, dbpool, user.ID)
430430 if err == nil {
431431 view.PostID = post.ID
432432 ch <- view
+0 -5 prose/config.go #
......@@ -17,14 +17,9 @@ func NewConfigSite() *shared.ConfigSite {
1717 dbURL := utils.GetEnv("DATABASE_URL", "")
1818 maxSize := uint64(500 * utils.MB)
1919 maxImgSize := int64(10 * utils.MB)
20- secret := utils.GetEnv("PICO_SECRET", "")
21- if secret == "" {
22- panic("must provide PICO_SECRET environment variable")
23- }
2420
2521 return &shared.ConfigSite{
2622 Debug: debug == "1",
27- Secret: secret,
2823 Domain: domain,
2924 Port: port,
3025 Protocol: protocol,
+53 -16 shared/analytics.go #
......@@ -25,8 +25,7 @@ func HmacString(secret, data string) string {
2525 return hex.EncodeToString(dataHmac)
2626 }
2727
28-func trackableRequest(r *http.Request) error {
29- agent := r.UserAgent()
28+func trackableUserAgent(agent string) error {
3029 // dont store requests from bots
3130 if crawlerdetect.IsCrawler(agent) {
3231 return fmt.Errorf(
......@@ -37,6 +36,11 @@ func trackableRequest(r *http.Request) error {
3736 return nil
3837 }
3938
39+func trackableRequest(r *http.Request) error {
40+ agent := r.UserAgent()
41+ return trackableUserAgent(agent)
42+}
43+
4044 func cleanIpAddress(ip string) (string, error) {
4145 host, _, err := net.SplitHostPort(ip)
4246 if err != nil {
......@@ -52,7 +56,15 @@ func cleanIpAddress(ip string) (string, error) {
5256 return anonIp, err
5357 }
5458
55-func cleanUrl(r *http.Request) (string, string) {
59+func cleanUrl(orig string) (string, string) {
60+ u, err := url.Parse(orig)
61+ if err != nil {
62+ return "", ""
63+ }
64+ return u.Host, u.Path
65+}
66+
67+func cleanUrlFromRequest(r *http.Request) (string, string) {
5668 host := r.Header.Get("x-forwarded-host")
5769 if host == "" {
5870 host = r.URL.Host
......@@ -81,16 +93,35 @@ func cleanReferer(ref string) (string, error) {
8193
8294 var ErrAnalyticsDisabled = errors.New("owner does not have site analytics enabled")
8395
84-func AnalyticsVisitFromRequest(r *http.Request, dbpool db.DB, userID string, secret string) (*db.AnalyticsVisits, error) {
85- if !dbpool.HasFeatureForUser(userID, "analytics") {
86- return nil, ErrAnalyticsDisabled
96+func AnalyticsVisitFromVisit(visit *db.AnalyticsVisits, dbpool db.DB, secret string) error {
97+ if !dbpool.HasFeatureForUser(visit.UserID, "analytics") {
98+ return ErrAnalyticsDisabled
8799 }
88100
89- err := trackableRequest(r)
101+ err := trackableUserAgent(visit.UserAgent)
90102 if err != nil {
91- return nil, err
103+ return err
104+ }
105+
106+ ipAddress, err := cleanIpAddress(visit.IpAddress)
107+ if err != nil {
108+ return err
109+ }
110+ visit.IpAddress = HmacString(secret, ipAddress)
111+ _, path := cleanUrl(visit.Path)
112+ visit.Path = path
113+
114+ referer, err := cleanReferer(visit.Referer)
115+ if err != nil {
116+ return err
92117 }
118+ visit.Referer = referer
119+ visit.UserAgent = cleanUserAgent(visit.UserAgent)
93120
121+ return nil
122+}
123+
124+func ipFromRequest(r *http.Request) string {
94125 // https://caddyserver.com/docs/caddyfile/directives/reverse_proxy#defaults
95126 ipOrig := r.Header.Get("x-forwarded-for")
96127 if ipOrig == "" {
......@@ -103,24 +134,30 @@ func AnalyticsVisitFromRequest(r *http.Request, dbpool db.DB, userID string, sec
103134 ipOrig = sshCtx.RemoteAddr().String()
104135 }
105136 }
106- ipAddress, err := cleanIpAddress(ipOrig)
107- if err != nil {
108- return nil, err
137+
138+ return ipOrig
139+}
140+
141+func AnalyticsVisitFromRequest(r *http.Request, dbpool db.DB, userID string) (*db.AnalyticsVisits, error) {
142+ if !dbpool.HasFeatureForUser(userID, "analytics") {
143+ return nil, ErrAnalyticsDisabled
109144 }
110- host, path := cleanUrl(r)
111145
112- referer, err := cleanReferer(r.Referer())
146+ err := trackableRequest(r)
113147 if err != nil {
114148 return nil, err
115149 }
116150
151+ ipAddress := ipFromRequest(r)
152+ host, path := cleanUrlFromRequest(r)
153+
117154 return &db.AnalyticsVisits{
118155 UserID: userID,
119156 Host: host,
120157 Path: path,
121- IpAddress: HmacString(secret, ipAddress),
122- UserAgent: cleanUserAgent(r.UserAgent()),
123- Referer: referer,
158+ IpAddress: ipAddress,
159+ UserAgent: r.UserAgent(),
160+ Referer: r.Referer(),
124161 Status: http.StatusOK,
125162 }, nil
126163 }
+0 -1 shared/config.go #
......@@ -30,7 +30,6 @@ type PageData struct {
3030 type ConfigSite struct {
3131 Debug bool
3232 SendgridKey string
33- Secret string
3433 Domain string
3534 Port string
3635 PortOverride string
+1 -0 sql/migrations/20241114_add_namespace_to_analytics.sql #
......@@ -0,0 +1,1 @@
1+ALTER TABLE analytics_visits ADD COLUMN namespace varchar(256);
Back to top