feat(auth): subscribe to pico's metric-drain pipe

Logs

erock created pr with ps-66 on 2024-11-12T21:13:35Z

erock added ps-73 on 2024-11-14T15:42:32Z

erock changed status on 2024-11-23T03:34:16Z {"status":"accepted"}

Patchsets

ps-66 by erock on 2024-11-12T21:13:35Z

Range Diff ↕


                1:  7ec3569 = 1:  7ec3569 feat(auth): subscribe to pico's metric-drain pipe


                2:  8a197f0 = 2:  8a197f0 chore: update pubsub


                3:  d4bda15 = 3:  d4bda15 refactor: use pipe for analytics


                -:  ------- > 4:  2b7c358 chore: prep for release

ps-73 by erock on 2024-11-14T15:42:32Z

Patchset `ps-73`

feat(auth): subscribe to pico's metric-drain pipe

Eric Bower

2024-11-11T02:41:27Z

auth/api.go

+33 -1

db/db.go

+10 -10

pico/cli.go

+2 -7

shared/config.go

+2 -7

shared/pubsub.go

+16 -0

tui/logs/logs.go

+3 -8

chore: update pubsub

Eric Bower

2024-11-12T16:29:38Z

go.mod

+2 -2

go.sum

+2 -2

refactor: use pipe for analytics

Eric Bower

2024-11-12T21:12:05Z

shared/analytics.go

+13 -3

chore: prep for release

Eric Bower

2024-11-14T15:10:28Z

Makefile

+2 -1

auth/api.go

+16 -2

db/db.go

+1 -0

db/postgres/storage.go

+12 -11

go.mod

+1 -1

go.sum

+2 -2

pgs/config.go

+0 -5

pgs/web_asset_handler.go

+2 -2

prose/api.go

+2 -2

prose/config.go

+0 -5

shared/analytics.go

+53 -16

shared/config.go

+0 -1

sql/migrations/20241114_add_namespace_to_analytics.sql

+1 -0

feat(auth): subscribe to pico's metric-drain pipe

We have a lot of services that need to record site usage analytics so we
need a distributed way to receive these events.

This overloads the auth service since it's now serving as a destination
for our metric-drain.

auth/api.go link

+33 -1

 1diff --git a/auth/api.go b/auth/api.go
 2index 8e279c4..8fdcaa2 100644
 3--- a/auth/api.go
 4+++ b/auth/api.go
 5@@ -1,6 +1,7 @@
 6 package auth
 7 
 8 import (
 9+	"bufio"
10 	"context"
11 	"crypto/hmac"
12 	"encoding/json"
13@@ -18,6 +19,7 @@ import (
14 	"github.com/picosh/pico/db"
15 	"github.com/picosh/pico/db/postgres"
16 	"github.com/picosh/pico/shared"
17+	"github.com/picosh/pubsub"
18 	"github.com/picosh/utils"
19 )
20 
21@@ -639,6 +641,31 @@ func handler(routes []shared.Route, client *Client) http.HandlerFunc {
22 	}
23 }
24 
25+func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger) {
26+	conn := shared.NewPicoPipeClient()
27+	stdoutPipe, err := pubsub.RemoteSub("sub metric-drain -k", ctx, conn)
28+
29+	if err != nil {
30+		logger.Error("could not sub to metric-drain", "err", err)
31+		return
32+	}
33+
34+	scanner := bufio.NewScanner(stdoutPipe)
35+	for scanner.Scan() {
36+		line := scanner.Text()
37+		view := db.AnalyticsVisits{}
38+		err := json.Unmarshal([]byte(line), &view)
39+		if err != nil {
40+			logger.Error("json unmarshal", "err", err)
41+			continue
42+		}
43+		err = dbpool.InsertVisit(&view)
44+		if err != nil {
45+			logger.Error("could not insert view record", "err", err)
46+		}
47+	}
48+}
49+
50 type AuthCfg struct {
51 	Debug  bool
52 	Port   string
53@@ -667,6 +694,11 @@ func StartApiServer() {
54 		Logger: logger,
55 	}
56 
57+	ctx := context.Background()
58+	// gather metrics in the auth service
59+	go metricDrainSub(ctx, db, logger)
60+	defer ctx.Done()
61+
62 	routes := createMainRoutes()
63 
64 	if cfg.Debug {
65@@ -679,6 +711,6 @@ func StartApiServer() {
66 	client.Logger.Info("starting server on port", "port", cfg.Port)
67 	err := http.ListenAndServe(portStr, router)
68 	if err != nil {
69-		client.Logger.Info(err.Error())
70+		client.Logger.Info("http-serve", "err", err.Error())
71 	}
72 }

db/db.go link

+10 -10

 1diff --git a/db/db.go b/db/db.go
 2index 4806b8d..fbcc715 100644
 3--- a/db/db.go
 4+++ b/db/db.go
 5@@ -161,16 +161,16 @@ type PostAnalytics struct {
 6 }
 7 
 8 type AnalyticsVisits struct {
 9-	ID        string
10-	UserID    string
11-	ProjectID string
12-	PostID    string
13-	Host      string
14-	Path      string
15-	IpAddress string
16-	UserAgent string
17-	Referer   string
18-	Status    int
19+	ID        string `json:"id"`
20+	UserID    string `json:"user_id"`
21+	ProjectID string `json:"project_id"`
22+	PostID    string `json:"post_id"`
23+	Host      string `json:"host"`
24+	Path      string `json:"path"`
25+	IpAddress string `json:"ip_adress"`
26+	UserAgent string `json:"user_agent"`
27+	Referer   string `json:"referer"`
28+	Status    int    `json:"status"`
29 }
30 
31 type VisitInterval struct {

pico/cli.go link

+2 -7

 1diff --git a/pico/cli.go b/pico/cli.go
 2index e64f0e4..ab65548 100644
 3--- a/pico/cli.go
 4+++ b/pico/cli.go
 5@@ -70,13 +70,8 @@ func (c *Cmd) notifications() error {
 6 }
 7 
 8 func (c *Cmd) logs(ctx context.Context) error {
 9-	stdoutPipe, err := pipeLogger.ConnectToLogs(ctx, &pipeLogger.PubSubConnectionInfo{
10-		RemoteHost:     utils.GetEnv("PICO_PIPE_ENDPOINT", "pipe.pico.sh:22"),
11-		KeyLocation:    utils.GetEnv("PICO_PIPE_KEY", "ssh_data/term_info_ed25519"),
12-		KeyPassphrase:  utils.GetEnv("PICO_PIPE_PASSPHRASE", ""),
13-		RemoteHostname: utils.GetEnv("PICO_PIPE_REMOTE_HOST", "pipe.pico.sh"),
14-		RemoteUser:     utils.GetEnv("PICO_PIPE_USER", "pico"),
15-	})
16+	conn := shared.NewPicoPipeClient()
17+	stdoutPipe, err := pipeLogger.ConnectToLogs(ctx, conn)
18 
19 	if err != nil {
20 		return err

shared/config.go link

+2 -7

 1diff --git a/shared/config.go b/shared/config.go
 2index 0413e1e..f1ad895 100644
 3--- a/shared/config.go
 4+++ b/shared/config.go
 5@@ -279,13 +279,8 @@ func CreateLogger(space string) *slog.Logger {
 6 	newLogger := log
 7 
 8 	if strings.ToLower(utils.GetEnv("PICO_PIPE_ENABLED", "true")) == "true" {
 9-		newLog, err := pipeLogger.SendLogRegister(log, &pipeLogger.PubSubConnectionInfo{
10-			RemoteHost:     utils.GetEnv("PICO_PIPE_ENDPOINT", "pipe.pico.sh:22"),
11-			KeyLocation:    utils.GetEnv("PICO_PIPE_KEY", "ssh_data/term_info_ed25519"),
12-			KeyPassphrase:  utils.GetEnv("PICO_PIPE_PASSPHRASE", ""),
13-			RemoteHostname: utils.GetEnv("PICO_PIPE_REMOTE_HOST", "pipe.pico.sh"),
14-			RemoteUser:     utils.GetEnv("PICO_PIPE_USER", "pico"),
15-		}, 100)
16+		conn := NewPicoPipeClient()
17+		newLog, err := pipeLogger.SendLogRegister(log, conn, 100)
18 
19 		if err == nil {
20 			newLogger = newLog

shared/pubsub.go link

+16 -0

 1diff --git a/shared/pubsub.go b/shared/pubsub.go
 2new file mode 100644
 3index 0000000..e0d9e73
 4--- /dev/null
 5+++ b/shared/pubsub.go
 6@@ -0,0 +1,16 @@
 7+package shared
 8+
 9+import (
10+	"github.com/picosh/pubsub"
11+	"github.com/picosh/utils"
12+)
13+
14+func NewPicoPipeClient() *pubsub.RemoteClientInfo {
15+	return &pubsub.RemoteClientInfo{
16+		RemoteHost:     utils.GetEnv("PICO_PIPE_ENDPOINT", "pipe.pico.sh:22"),
17+		KeyLocation:    utils.GetEnv("PICO_PIPE_KEY", "ssh_data/term_info_ed25519"),
18+		KeyPassphrase:  utils.GetEnv("PICO_PIPE_PASSPHRASE", ""),
19+		RemoteHostname: utils.GetEnv("PICO_PIPE_REMOTE_HOST", "pipe.pico.sh"),
20+		RemoteUser:     utils.GetEnv("PICO_PIPE_USER", "pico"),
21+	}
22+}

tui/logs/logs.go link

+3 -8

 1diff --git a/tui/logs/logs.go b/tui/logs/logs.go
 2index d2f2891..6e01c7c 100644
 3--- a/tui/logs/logs.go
 4+++ b/tui/logs/logs.go
 5@@ -11,6 +11,7 @@ import (
 6 	"github.com/charmbracelet/bubbles/viewport"
 7 	tea "github.com/charmbracelet/bubbletea"
 8 	"github.com/charmbracelet/lipgloss"
 9+	"github.com/picosh/pico/shared"
10 	"github.com/picosh/pico/tui/common"
11 	"github.com/picosh/pico/tui/pages"
12 	"github.com/picosh/utils"
13@@ -170,14 +171,8 @@ func (m Model) waitForActivity(sub chan map[string]any) tea.Cmd {
14 
15 func (m Model) connectLogs(sub chan map[string]any) tea.Cmd {
16 	return func() tea.Msg {
17-		stdoutPipe, err := pipeLogger.ConnectToLogs(m.ctx, &pipeLogger.PubSubConnectionInfo{
18-			RemoteHost:     utils.GetEnv("PICO_PIPE_ENDPOINT", "pipe.pico.sh:22"),
19-			KeyLocation:    utils.GetEnv("PICO_PIPE_KEY", "ssh_data/term_info_ed25519"),
20-			KeyPassphrase:  utils.GetEnv("PICO_PIPE_PASSPHRASE", ""),
21-			RemoteHostname: utils.GetEnv("PICO_PIPE_REMOTE_HOST", "pipe.pico.sh"),
22-			RemoteUser:     utils.GetEnv("PICO_PIPE_USER", "pico"),
23-		})
24-
25+		conn := shared.NewPicoPipeClient()
26+		stdoutPipe, err := pipeLogger.ConnectToLogs(m.ctx, conn)
27 		if err != nil {
28 			return errMsg(err)
29 		}

chore: update pubsub

go.mod link

+2 -2

 1diff --git a/go.mod b/go.mod
 2index 90c2f02..b588d80 100644
 3--- a/go.mod
 4+++ b/go.mod
 5@@ -31,13 +31,12 @@ require (
 6 	github.com/gorilla/feeds v1.2.0
 7 	github.com/lib/pq v1.10.9
 8 	github.com/microcosm-cc/bluemonday v1.0.27
 9-	github.com/minio/minio-go/v7 v7.0.80
10 	github.com/mmcdole/gofeed v1.3.0
11 	github.com/muesli/reflow v0.3.0
12 	github.com/muesli/termenv v0.15.3-0.20240912151726-82936c5ea257
13 	github.com/neurosnap/go-exif-remove v0.0.0-20221010134343-50d1e3c35577
14 	github.com/picosh/pobj v0.0.0-20241016194248-c39198b2ff23
15-	github.com/picosh/pubsub v0.0.0-20241030185810-e24d08b67ab8
16+	github.com/picosh/pubsub v0.0.0-20241112151357-866d44c53659
17 	github.com/picosh/send v0.0.0-20241107150437-0febb0049b4f
18 	github.com/picosh/tunkit v0.0.0-20240905223921-532404cef9d9
19 	github.com/picosh/utils v0.0.0-20241018143404-b351d5d765f3
20@@ -133,6 +132,7 @@ require (
21 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
22 	github.com/minio/madmin-go/v3 v3.0.75 // indirect
23 	github.com/minio/md5-simd v1.1.2 // indirect
24+	github.com/minio/minio-go/v7 v7.0.80 // indirect
25 	github.com/mmcdole/goxpp v1.1.1 // indirect
26 	github.com/mmcloughlin/md4 v0.1.2 // indirect
27 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

go.sum link

+2 -2

 1diff --git a/go.sum b/go.sum
 2index 4fc24c3..c70d06b 100644
 3--- a/go.sum
 4+++ b/go.sum
 5@@ -269,8 +269,8 @@ github.com/picosh/go-rsync-receiver v0.0.0-20240709135253-1daf4b12a9fc h1:bvcsoO
 6 github.com/picosh/go-rsync-receiver v0.0.0-20240709135253-1daf4b12a9fc/go.mod h1:i0iR3W4GSm1PuvVxB9OH32E5jP+CYkVb2NQSe0JCtlo=
 7 github.com/picosh/pobj v0.0.0-20241016194248-c39198b2ff23 h1:NEJ5a4UXeF0/X7xmYNzXcwLQID9DwgazlqkMMC5zZ3M=
 8 github.com/picosh/pobj v0.0.0-20241016194248-c39198b2ff23/go.mod h1:cF+eAl4G1vU+WOD8cYCKaxokHo6MWmbR8J4/SJnvESg=
 9-github.com/picosh/pubsub v0.0.0-20241030185810-e24d08b67ab8 h1:E/eQsxdHBctPArAzjSHUAVZtDXjsD1AduGD94mbUJQg=
10-github.com/picosh/pubsub v0.0.0-20241030185810-e24d08b67ab8/go.mod h1:ajolgob5MxlHdp5HllF7u3rTlCgER4InqfP7M/xl6HQ=
11+github.com/picosh/pubsub v0.0.0-20241112151357-866d44c53659 h1:HmRi+QkAcKkOcLD90xbf7qZy95muQEd/DqttK9xtpHk=
12+github.com/picosh/pubsub v0.0.0-20241112151357-866d44c53659/go.mod h1:m6ZZpg+lZB3XTIKlbSqQgi4NrBPtARv23b8vGYDoCo4=
13 github.com/picosh/send v0.0.0-20241107150437-0febb0049b4f h1:pdEh1Z7zH5Og9nS7jRuqwup3bcPsC6faDNQ6mgrV9ws=
14 github.com/picosh/send v0.0.0-20241107150437-0febb0049b4f/go.mod h1:RAgLDK3LrDK6pNeXtU9tjo28obl5DxShcTUk2nm/KCM=
15 github.com/picosh/senpai v0.0.0-20240503200611-af89e73973b0 h1:pBRIbiCj7K6rGELijb//dYhyCo8A3fvxW5dijrJVtjs=

refactor: use pipe for analytics

shared/analytics.go link

+13 -3

 1diff --git a/shared/analytics.go b/shared/analytics.go
 2index cfe69bc..136e347 100644
 3--- a/shared/analytics.go
 4+++ b/shared/analytics.go
 5@@ -4,6 +4,7 @@ import (
 6 	"crypto/hmac"
 7 	"crypto/sha256"
 8 	"encoding/hex"
 9+	"encoding/json"
10 	"errors"
11 	"fmt"
12 	"log/slog"
13@@ -12,6 +13,7 @@ import (
14 	"net/url"
15 
16 	"github.com/picosh/pico/db"
17+	"github.com/picosh/pubsub"
18 	"github.com/simplesurance/go-ip-anonymizer/ipanonymizer"
19 	"github.com/x-way/crawlerdetect"
20 )
21@@ -124,10 +126,18 @@ func AnalyticsVisitFromRequest(r *http.Request, dbpool db.DB, userID string, sec
22 }
23 
24 func AnalyticsCollect(ch chan *db.AnalyticsVisits, dbpool db.DB, logger *slog.Logger) {
25-	for view := range ch {
26-		err := dbpool.InsertVisit(view)
27+	info := NewPicoPipeClient()
28+	metricDrain := pubsub.NewRemoteClientWriter(info, logger, 0)
29+	go metricDrain.KeepAlive("pub metric-drain -b=false")
30+
31+	for visit := range ch {
32+		data, err := json.Marshal(visit)
33+		if err != nil {
34+			logger.Error("could not json marshall visit record", "err", err)
35+		}
36+		_, err = metricDrain.Write(data)
37 		if err != nil {
38-			logger.Error("could not insert view record", "err", err)
39+			logger.Error("could not write to metric-drain", "err", err)
40 		}
41 	}
42 }

chore: prep for release

Makefile link

+2 -1

 1diff --git a/Makefile b/Makefile
 2index 8cc530f..963fc9d 100644
 3--- a/Makefile
 4+++ b/Makefile
 5@@ -130,10 +130,11 @@ migrate:
 6 	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20240324_add_analytics_table.sql
 7 	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20240819_add_projects_blocked.sql
 8 	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241028_add_analytics_indexes.sql
 9+	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241114_add_namespace_to_analytics.sql
10 .PHONY: migrate
11 
12 latest:
13-	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241028_add_analytics_indexes.sql
14+	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241114_add_namespace_to_analytics.sql
15 .PHONY: latest
16 
17 psql:

auth/api.go link

+16 -2

 1diff --git a/auth/api.go b/auth/api.go
 2index 8fdcaa2..c7b80d9 100644
 3--- a/auth/api.go
 4+++ b/auth/api.go
 5@@ -5,6 +5,7 @@ import (
 6 	"context"
 7 	"crypto/hmac"
 8 	"encoding/json"
 9+	"errors"
10 	"fmt"
11 	"html/template"
12 	"io"
13@@ -641,7 +642,7 @@ func handler(routes []shared.Route, client *Client) http.HandlerFunc {
14 	}
15 }
16 
17-func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger) {
18+func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secret string) {
19 	conn := shared.NewPicoPipeClient()
20 	stdoutPipe, err := pubsub.RemoteSub("sub metric-drain -k", ctx, conn)
21 
22@@ -659,6 +660,14 @@ func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger) {
23 			logger.Error("json unmarshal", "err", err)
24 			continue
25 		}
26+
27+		err = shared.AnalyticsVisitFromVisit(&view, dbpool, secret)
28+		if err != nil {
29+			if !errors.Is(err, shared.ErrAnalyticsDisabled) {
30+				logger.Info("could not record analytics view", "reason", err)
31+			}
32+		}
33+
34 		err = dbpool.InsertVisit(&view)
35 		if err != nil {
36 			logger.Error("could not insert view record", "err", err)
37@@ -672,6 +681,7 @@ type AuthCfg struct {
38 	DbURL  string
39 	Domain string
40 	Issuer string
41+	Secret string
42 }
43 
44 func StartApiServer() {
45@@ -682,6 +692,10 @@ func StartApiServer() {
46 		Issuer: utils.GetEnv("AUTH_ISSUER", "pico.sh"),
47 		Domain: utils.GetEnv("AUTH_DOMAIN", "http://0.0.0.0:3000"),
48 		Port:   utils.GetEnv("AUTH_WEB_PORT", "3000"),
49+		Secret: utils.GetEnv("PICO_SECRET", ""),
50+	}
51+	if cfg.Secret == "" {
52+		panic("must provide PICO_SECRET environment variable")
53 	}
54 
55 	logger := shared.CreateLogger("auth")
56@@ -696,7 +710,7 @@ func StartApiServer() {
57 
58 	ctx := context.Background()
59 	// gather metrics in the auth service
60-	go metricDrainSub(ctx, db, logger)
61+	go metricDrainSub(ctx, db, logger, cfg.Secret)
62 	defer ctx.Done()
63 
64 	routes := createMainRoutes()

db/db.go link

+1 -0

 1diff --git a/db/db.go b/db/db.go
 2index fbcc715..e2d03d1 100644
 3--- a/db/db.go
 4+++ b/db/db.go
 5@@ -165,6 +165,7 @@ type AnalyticsVisits struct {
 6 	UserID    string `json:"user_id"`
 7 	ProjectID string `json:"project_id"`
 8 	PostID    string `json:"post_id"`
 9+	Namespace string `json:"namespace"`
10 	Host      string `json:"host"`
11 	Path      string `json:"path"`
12 	IpAddress string `json:"ip_adress"`

db/postgres/storage.go link

+12 -11

 1diff --git a/db/postgres/storage.go b/db/postgres/storage.go
 2index 04975d7..6440cfc 100644
 3--- a/db/postgres/storage.go
 4+++ b/db/postgres/storage.go
 5@@ -984,18 +984,19 @@ func newNullString(s string) sql.NullString {
 6 	}
 7 }
 8 
 9-func (me *PsqlDB) InsertVisit(view *db.AnalyticsVisits) error {
10+func (me *PsqlDB) InsertVisit(visit *db.AnalyticsVisits) error {
11 	_, err := me.Db.Exec(
12-		`INSERT INTO analytics_visits (user_id, project_id, post_id, host, path, ip_address, user_agent, referer, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9);`,
13-		view.UserID,
14-		newNullString(view.ProjectID),
15-		newNullString(view.PostID),
16-		view.Host,
17-		view.Path,
18-		view.IpAddress,
19-		view.UserAgent,
20-		view.Referer,
21-		view.Status,
22+		`INSERT INTO analytics_visits (user_id, project_id, post_id, namespace, host, path, ip_address, user_agent, referer, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);`,
23+		visit.UserID,
24+		newNullString(visit.ProjectID),
25+		newNullString(visit.PostID),
26+		newNullString(visit.Namespace),
27+		visit.Host,
28+		visit.Path,
29+		visit.IpAddress,
30+		visit.UserAgent,
31+		visit.Referer,
32+		visit.Status,
33 	)
34 	return err
35 }

go.mod link

+1 -1

 1diff --git a/go.mod b/go.mod
 2index b588d80..d5a954d 100644
 3--- a/go.mod
 4+++ b/go.mod
 5@@ -36,7 +36,7 @@ require (
 6 	github.com/muesli/termenv v0.15.3-0.20240912151726-82936c5ea257
 7 	github.com/neurosnap/go-exif-remove v0.0.0-20221010134343-50d1e3c35577
 8 	github.com/picosh/pobj v0.0.0-20241016194248-c39198b2ff23
 9-	github.com/picosh/pubsub v0.0.0-20241112151357-866d44c53659
10+	github.com/picosh/pubsub v0.0.0-20241114025640-35db438302b4
11 	github.com/picosh/send v0.0.0-20241107150437-0febb0049b4f
12 	github.com/picosh/tunkit v0.0.0-20240905223921-532404cef9d9
13 	github.com/picosh/utils v0.0.0-20241018143404-b351d5d765f3

go.sum link

+2 -2

 1diff --git a/go.sum b/go.sum
 2index c70d06b..b8c229a 100644
 3--- a/go.sum
 4+++ b/go.sum
 5@@ -269,8 +269,8 @@ github.com/picosh/go-rsync-receiver v0.0.0-20240709135253-1daf4b12a9fc h1:bvcsoO
 6 github.com/picosh/go-rsync-receiver v0.0.0-20240709135253-1daf4b12a9fc/go.mod h1:i0iR3W4GSm1PuvVxB9OH32E5jP+CYkVb2NQSe0JCtlo=
 7 github.com/picosh/pobj v0.0.0-20241016194248-c39198b2ff23 h1:NEJ5a4UXeF0/X7xmYNzXcwLQID9DwgazlqkMMC5zZ3M=
 8 github.com/picosh/pobj v0.0.0-20241016194248-c39198b2ff23/go.mod h1:cF+eAl4G1vU+WOD8cYCKaxokHo6MWmbR8J4/SJnvESg=
 9-github.com/picosh/pubsub v0.0.0-20241112151357-866d44c53659 h1:HmRi+QkAcKkOcLD90xbf7qZy95muQEd/DqttK9xtpHk=
10-github.com/picosh/pubsub v0.0.0-20241112151357-866d44c53659/go.mod h1:m6ZZpg+lZB3XTIKlbSqQgi4NrBPtARv23b8vGYDoCo4=
11+github.com/picosh/pubsub v0.0.0-20241114025640-35db438302b4 h1:pITSRXb9NDGdC6AmuS3JE+8Ek4/pUG7tXJPP3cOaqf4=
12+github.com/picosh/pubsub v0.0.0-20241114025640-35db438302b4/go.mod h1:m6ZZpg+lZB3XTIKlbSqQgi4NrBPtARv23b8vGYDoCo4=
13 github.com/picosh/send v0.0.0-20241107150437-0febb0049b4f h1:pdEh1Z7zH5Og9nS7jRuqwup3bcPsC6faDNQ6mgrV9ws=
14 github.com/picosh/send v0.0.0-20241107150437-0febb0049b4f/go.mod h1:RAgLDK3LrDK6pNeXtU9tjo28obl5DxShcTUk2nm/KCM=
15 github.com/picosh/senpai v0.0.0-20240503200611-af89e73973b0 h1:pBRIbiCj7K6rGELijb//dYhyCo8A3fvxW5dijrJVtjs=

pgs/config.go link

+0 -5

 1diff --git a/pgs/config.go b/pgs/config.go
 2index 3cc9a57..f0cdaa0 100644
 3--- a/pgs/config.go
 4+++ b/pgs/config.go
 5@@ -20,13 +20,8 @@ func NewConfigSite() *shared.ConfigSite {
 6 	minioUser := utils.GetEnv("MINIO_ROOT_USER", "")
 7 	minioPass := utils.GetEnv("MINIO_ROOT_PASSWORD", "")
 8 	dbURL := utils.GetEnv("DATABASE_URL", "")
 9-	secret := utils.GetEnv("PICO_SECRET", "")
10-	if secret == "" {
11-		panic("must provide PICO_SECRET environment variable")
12-	}
13 
14 	cfg := shared.ConfigSite{
15-		Secret:             secret,
16 		Domain:             domain,
17 		Port:               port,
18 		Protocol:           protocol,

pgs/web_asset_handler.go link

+2 -2

 1diff --git a/pgs/web_asset_handler.go b/pgs/web_asset_handler.go
 2index 7508702..3ff5521 100644
 3--- a/pgs/web_asset_handler.go
 4+++ b/pgs/web_asset_handler.go
 5@@ -157,7 +157,7 @@ func (h *ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 6 		)
 7 		// track 404s
 8 		ch := h.AnalyticsQueue
 9-		view, err := shared.AnalyticsVisitFromRequest(r, h.Dbpool, h.UserID, h.Cfg.Secret)
10+		view, err := shared.AnalyticsVisitFromRequest(r, h.Dbpool, h.UserID)
11 		if err == nil {
12 			view.ProjectID = h.ProjectID
13 			view.Status = http.StatusNotFound
14@@ -236,7 +236,7 @@ func (h *ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
15 	if finContentType == "text/html" {
16 		// track visit
17 		ch := h.AnalyticsQueue
18-		view, err := shared.AnalyticsVisitFromRequest(r, h.Dbpool, h.UserID, h.Cfg.Secret)
19+		view, err := shared.AnalyticsVisitFromRequest(r, h.Dbpool, h.UserID)
20 		if err == nil {
21 			view.ProjectID = h.ProjectID
22 			ch <- view

prose/api.go link

+2 -2

 1diff --git a/prose/api.go b/prose/api.go
 2index 133ec68..402cc67 100644
 3--- a/prose/api.go
 4+++ b/prose/api.go
 5@@ -272,7 +272,7 @@ func blogHandler(w http.ResponseWriter, r *http.Request) {
 6 
 7 	// track visit
 8 	ch := shared.GetAnalyticsQueue(r)
 9-	view, err := shared.AnalyticsVisitFromRequest(r, dbpool, user.ID, cfg.Secret)
10+	view, err := shared.AnalyticsVisitFromRequest(r, dbpool, user.ID)
11 	if err == nil {
12 		ch <- view
13 	} else {
14@@ -426,7 +426,7 @@ func postHandler(w http.ResponseWriter, r *http.Request) {
15 		}
16 
17 		// track visit
18-		view, err := shared.AnalyticsVisitFromRequest(r, dbpool, user.ID, cfg.Secret)
19+		view, err := shared.AnalyticsVisitFromRequest(r, dbpool, user.ID)
20 		if err == nil {
21 			view.PostID = post.ID
22 			ch <- view

prose/config.go link

+0 -5

 1diff --git a/prose/config.go b/prose/config.go
 2index 878eb9b..b1a02b0 100644
 3--- a/prose/config.go
 4+++ b/prose/config.go
 5@@ -17,14 +17,9 @@ func NewConfigSite() *shared.ConfigSite {
 6 	dbURL := utils.GetEnv("DATABASE_URL", "")
 7 	maxSize := uint64(500 * utils.MB)
 8 	maxImgSize := int64(10 * utils.MB)
 9-	secret := utils.GetEnv("PICO_SECRET", "")
10-	if secret == "" {
11-		panic("must provide PICO_SECRET environment variable")
12-	}
13 
14 	return &shared.ConfigSite{
15 		Debug:      debug == "1",
16-		Secret:     secret,
17 		Domain:     domain,
18 		Port:       port,
19 		Protocol:   protocol,

shared/analytics.go link

+53 -16

  1diff --git a/shared/analytics.go b/shared/analytics.go
  2index 136e347..a855007 100644
  3--- a/shared/analytics.go
  4+++ b/shared/analytics.go
  5@@ -25,8 +25,7 @@ func HmacString(secret, data string) string {
  6 	return hex.EncodeToString(dataHmac)
  7 }
  8 
  9-func trackableRequest(r *http.Request) error {
 10-	agent := r.UserAgent()
 11+func trackableUserAgent(agent string) error {
 12 	// dont store requests from bots
 13 	if crawlerdetect.IsCrawler(agent) {
 14 		return fmt.Errorf(
 15@@ -37,6 +36,11 @@ func trackableRequest(r *http.Request) error {
 16 	return nil
 17 }
 18 
 19+func trackableRequest(r *http.Request) error {
 20+	agent := r.UserAgent()
 21+	return trackableUserAgent(agent)
 22+}
 23+
 24 func cleanIpAddress(ip string) (string, error) {
 25 	host, _, err := net.SplitHostPort(ip)
 26 	if err != nil {
 27@@ -52,7 +56,15 @@ func cleanIpAddress(ip string) (string, error) {
 28 	return anonIp, err
 29 }
 30 
 31-func cleanUrl(r *http.Request) (string, string) {
 32+func cleanUrl(orig string) (string, string) {
 33+	u, err := url.Parse(orig)
 34+	if err != nil {
 35+		return "", ""
 36+	}
 37+	return u.Host, u.Path
 38+}
 39+
 40+func cleanUrlFromRequest(r *http.Request) (string, string) {
 41 	host := r.Header.Get("x-forwarded-host")
 42 	if host == "" {
 43 		host = r.URL.Host
 44@@ -81,16 +93,35 @@ func cleanReferer(ref string) (string, error) {
 45 
 46 var ErrAnalyticsDisabled = errors.New("owner does not have site analytics enabled")
 47 
 48-func AnalyticsVisitFromRequest(r *http.Request, dbpool db.DB, userID string, secret string) (*db.AnalyticsVisits, error) {
 49-	if !dbpool.HasFeatureForUser(userID, "analytics") {
 50-		return nil, ErrAnalyticsDisabled
 51+func AnalyticsVisitFromVisit(visit *db.AnalyticsVisits, dbpool db.DB, secret string) error {
 52+	if !dbpool.HasFeatureForUser(visit.UserID, "analytics") {
 53+		return ErrAnalyticsDisabled
 54 	}
 55 
 56-	err := trackableRequest(r)
 57+	err := trackableUserAgent(visit.UserAgent)
 58 	if err != nil {
 59-		return nil, err
 60+		return err
 61+	}
 62+
 63+	ipAddress, err := cleanIpAddress(visit.IpAddress)
 64+	if err != nil {
 65+		return err
 66+	}
 67+	visit.IpAddress = HmacString(secret, ipAddress)
 68+	_, path := cleanUrl(visit.Path)
 69+	visit.Path = path
 70+
 71+	referer, err := cleanReferer(visit.Referer)
 72+	if err != nil {
 73+		return err
 74 	}
 75+	visit.Referer = referer
 76+	visit.UserAgent = cleanUserAgent(visit.UserAgent)
 77 
 78+	return nil
 79+}
 80+
 81+func ipFromRequest(r *http.Request) string {
 82 	// https://caddyserver.com/docs/caddyfile/directives/reverse_proxy#defaults
 83 	ipOrig := r.Header.Get("x-forwarded-for")
 84 	if ipOrig == "" {
 85@@ -103,24 +134,30 @@ func AnalyticsVisitFromRequest(r *http.Request, dbpool db.DB, userID string, sec
 86 			ipOrig = sshCtx.RemoteAddr().String()
 87 		}
 88 	}
 89-	ipAddress, err := cleanIpAddress(ipOrig)
 90-	if err != nil {
 91-		return nil, err
 92+
 93+	return ipOrig
 94+}
 95+
 96+func AnalyticsVisitFromRequest(r *http.Request, dbpool db.DB, userID string) (*db.AnalyticsVisits, error) {
 97+	if !dbpool.HasFeatureForUser(userID, "analytics") {
 98+		return nil, ErrAnalyticsDisabled
 99 	}
100-	host, path := cleanUrl(r)
101 
102-	referer, err := cleanReferer(r.Referer())
103+	err := trackableRequest(r)
104 	if err != nil {
105 		return nil, err
106 	}
107 
108+	ipAddress := ipFromRequest(r)
109+	host, path := cleanUrlFromRequest(r)
110+
111 	return &db.AnalyticsVisits{
112 		UserID:    userID,
113 		Host:      host,
114 		Path:      path,
115-		IpAddress: HmacString(secret, ipAddress),
116-		UserAgent: cleanUserAgent(r.UserAgent()),
117-		Referer:   referer,
118+		IpAddress: ipAddress,
119+		UserAgent: r.UserAgent(),
120+		Referer:   r.Referer(),
121 		Status:    http.StatusOK,
122 	}, nil
123 }

shared/config.go link

+0 -1

 1diff --git a/shared/config.go b/shared/config.go
 2index f1ad895..daff8d5 100644
 3--- a/shared/config.go
 4+++ b/shared/config.go
 5@@ -30,7 +30,6 @@ type PageData struct {
 6 type ConfigSite struct {
 7 	Debug              bool
 8 	SendgridKey        string
 9-	Secret             string
10 	Domain             string
11 	Port               string
12 	PortOverride       string

sql/migrations/20241114_add_namespace_to_analytics.sql link

+1 -0

1diff --git a/sql/migrations/20241114_add_namespace_to_analytics.sql b/sql/migrations/20241114_add_namespace_to_analytics.sql
2new file mode 100644
3index 0000000..2ff6618
4--- /dev/null
5+++ b/sql/migrations/20241114_add_namespace_to_analytics.sql
6@@ -0,0 +1,1 @@
7+ALTER TABLE analytics_visits ADD COLUMN namespace varchar(256);

Logs

Patchsets

Patchset ps-73

Patchset `ps-73`