dashboard / erock/pico / feat(auth): subscribe to pico's metric-drain pipe #32 rss

old

new

 	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20240324_add_analytics_table.sql
 	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20240819_add_projects_blocked.sql
 	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241028_add_analytics_indexes.sql
+	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241114_add_namespace_to_analytics.sql
 .PHONY: migrate
 
 latest:
-	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241028_add_analytics_indexes.sql
+	$(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241114_add_namespace_to_analytics.sql
 .PHONY: latest
 
 psql:

old

new

 	"context"
 	"crypto/hmac"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"html/template"
 	"io"
 	}
 }
 
-func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger) {
+func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secret string) {
 	conn := shared.NewPicoPipeClient()
 	stdoutPipe, err := pubsub.RemoteSub("sub metric-drain -k", ctx, conn)
 
 			logger.Error("json unmarshal", "err", err)
 			continue
 		}
+
+		err = shared.AnalyticsVisitFromVisit(&view, dbpool, secret)
+		if err != nil {
+			if !errors.Is(err, shared.ErrAnalyticsDisabled) {
+				logger.Info("could not record analytics view", "reason", err)
+			}
+		}
+
 		err = dbpool.InsertVisit(&view)
 		if err != nil {
 			logger.Error("could not insert view record", "err", err)
 	DbURL  string
 	Domain string
 	Issuer string
+	Secret string
 }
 
 func StartApiServer() {
 		Issuer: utils.GetEnv("AUTH_ISSUER", "pico.sh"),
 		Domain: utils.GetEnv("AUTH_DOMAIN", "http://0.0.0.0:3000"),
 		Port:   utils.GetEnv("AUTH_WEB_PORT", "3000"),
+		Secret: utils.GetEnv("PICO_SECRET", ""),
+	}
+	if cfg.Secret == "" {
+		panic("must provide PICO_SECRET environment variable")
 	}
 
 	logger := shared.CreateLogger("auth")
 
 	ctx := context.Background()
 	// gather metrics in the auth service
-	go metricDrainSub(ctx, db, logger)
+	go metricDrainSub(ctx, db, logger, cfg.Secret)
 	defer ctx.Done()
 
 	routes := createMainRoutes()

old

new

 	UserID    string `json:"user_id"`
 	ProjectID string `json:"project_id"`
 	PostID    string `json:"post_id"`
+	Namespace string `json:"namespace"`
 	Host      string `json:"host"`
 	Path      string `json:"path"`
 	IpAddress string `json:"ip_adress"`

old

new

 	}
 }
 
-func (me *PsqlDB) InsertVisit(view *db.AnalyticsVisits) error {
+func (me *PsqlDB) InsertVisit(visit *db.AnalyticsVisits) error {
 	_, err := me.Db.Exec(
-		`INSERT INTO analytics_visits (user_id, project_id, post_id, host, path, ip_address, user_agent, referer, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9);`,
-		view.UserID,
-		newNullString(view.ProjectID),
-		newNullString(view.PostID),
-		view.Host,
-		view.Path,
-		view.IpAddress,
-		view.UserAgent,
-		view.Referer,
-		view.Status,
+		`INSERT INTO analytics_visits (user_id, project_id, post_id, namespace, host, path, ip_address, user_agent, referer, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);`,
+		visit.UserID,
+		newNullString(visit.ProjectID),
+		newNullString(visit.PostID),
+		newNullString(visit.Namespace),
+		visit.Host,
+		visit.Path,
+		visit.IpAddress,
+		visit.UserAgent,
+		visit.Referer,
+		visit.Status,
 	)
 	return err
 }

old

new

 	github.com/muesli/termenv v0.15.3-0.20240912151726-82936c5ea257
 	github.com/neurosnap/go-exif-remove v0.0.0-20221010134343-50d1e3c35577
 	github.com/picosh/pobj v0.0.0-20241016194248-c39198b2ff23
-	github.com/picosh/pubsub v0.0.0-20241112151357-866d44c53659
+	github.com/picosh/pubsub v0.0.0-20241114025640-35db438302b4
 	github.com/picosh/send v0.0.0-20241107150437-0febb0049b4f
 	github.com/picosh/tunkit v0.0.0-20240905223921-532404cef9d9
 	github.com/picosh/utils v0.0.0-20241018143404-b351d5d765f3

old

new

 github.com/picosh/go-rsync-receiver v0.0.0-20240709135253-1daf4b12a9fc/go.mod h1:i0iR3W4GSm1PuvVxB9OH32E5jP+CYkVb2NQSe0JCtlo=
 github.com/picosh/pobj v0.0.0-20241016194248-c39198b2ff23 h1:NEJ5a4UXeF0/X7xmYNzXcwLQID9DwgazlqkMMC5zZ3M=
 github.com/picosh/pobj v0.0.0-20241016194248-c39198b2ff23/go.mod h1:cF+eAl4G1vU+WOD8cYCKaxokHo6MWmbR8J4/SJnvESg=
-github.com/picosh/pubsub v0.0.0-20241112151357-866d44c53659 h1:HmRi+QkAcKkOcLD90xbf7qZy95muQEd/DqttK9xtpHk=
-github.com/picosh/pubsub v0.0.0-20241112151357-866d44c53659/go.mod h1:m6ZZpg+lZB3XTIKlbSqQgi4NrBPtARv23b8vGYDoCo4=
+github.com/picosh/pubsub v0.0.0-20241114025640-35db438302b4 h1:pITSRXb9NDGdC6AmuS3JE+8Ek4/pUG7tXJPP3cOaqf4=
+github.com/picosh/pubsub v0.0.0-20241114025640-35db438302b4/go.mod h1:m6ZZpg+lZB3XTIKlbSqQgi4NrBPtARv23b8vGYDoCo4=
 github.com/picosh/send v0.0.0-20241107150437-0febb0049b4f h1:pdEh1Z7zH5Og9nS7jRuqwup3bcPsC6faDNQ6mgrV9ws=
 github.com/picosh/send v0.0.0-20241107150437-0febb0049b4f/go.mod h1:RAgLDK3LrDK6pNeXtU9tjo28obl5DxShcTUk2nm/KCM=
 github.com/picosh/senpai v0.0.0-20240503200611-af89e73973b0 h1:pBRIbiCj7K6rGELijb//dYhyCo8A3fvxW5dijrJVtjs=

old

new

 	minioUser := utils.GetEnv("MINIO_ROOT_USER", "")
 	minioPass := utils.GetEnv("MINIO_ROOT_PASSWORD", "")
 	dbURL := utils.GetEnv("DATABASE_URL", "")
-	secret := utils.GetEnv("PICO_SECRET", "")
-	if secret == "" {
-		panic("must provide PICO_SECRET environment variable")
-	}
 
 	cfg := shared.ConfigSite{
-		Secret:             secret,
 		Domain:             domain,
 		Port:               port,
 		Protocol:           protocol,

old

new

 		)
 		// track 404s
 		ch := h.AnalyticsQueue
-		view, err := shared.AnalyticsVisitFromRequest(r, h.Dbpool, h.UserID, h.Cfg.Secret)
+		view, err := shared.AnalyticsVisitFromRequest(r, h.Dbpool, h.UserID)
 		if err == nil {
 			view.ProjectID = h.ProjectID
 			view.Status = http.StatusNotFound
 	if finContentType == "text/html" {
 		// track visit
 		ch := h.AnalyticsQueue
-		view, err := shared.AnalyticsVisitFromRequest(r, h.Dbpool, h.UserID, h.Cfg.Secret)
+		view, err := shared.AnalyticsVisitFromRequest(r, h.Dbpool, h.UserID)
 		if err == nil {
 			view.ProjectID = h.ProjectID
 			ch <- view

old

new

 
 	// track visit
 	ch := shared.GetAnalyticsQueue(r)
-	view, err := shared.AnalyticsVisitFromRequest(r, dbpool, user.ID, cfg.Secret)
+	view, err := shared.AnalyticsVisitFromRequest(r, dbpool, user.ID)
 	if err == nil {
 		ch <- view
 	} else {
 		}
 
 		// track visit
-		view, err := shared.AnalyticsVisitFromRequest(r, dbpool, user.ID, cfg.Secret)
+		view, err := shared.AnalyticsVisitFromRequest(r, dbpool, user.ID)
 		if err == nil {
 			view.PostID = post.ID
 			ch <- view

old

new

 	dbURL := utils.GetEnv("DATABASE_URL", "")
 	maxSize := uint64(500 * utils.MB)
 	maxImgSize := int64(10 * utils.MB)
-	secret := utils.GetEnv("PICO_SECRET", "")
-	if secret == "" {
-		panic("must provide PICO_SECRET environment variable")
-	}
 
 	return &shared.ConfigSite{
 		Debug:      debug == "1",
-		Secret:     secret,
 		Domain:     domain,
 		Port:       port,
 		Protocol:   protocol,

old

new

 	return hex.EncodeToString(dataHmac)
 }
 
-func trackableRequest(r *http.Request) error {
-	agent := r.UserAgent()
+func trackableUserAgent(agent string) error {
 	// dont store requests from bots
 	if crawlerdetect.IsCrawler(agent) {
 		return fmt.Errorf(
 	return nil
 }
 
+func trackableRequest(r *http.Request) error {
+	agent := r.UserAgent()
+	return trackableUserAgent(agent)
+}
+
 func cleanIpAddress(ip string) (string, error) {
 	host, _, err := net.SplitHostPort(ip)
 	if err != nil {
 	return anonIp, err
 }
 
-func cleanUrl(r *http.Request) (string, string) {
+func cleanUrl(orig string) (string, string) {
+	u, err := url.Parse(orig)
+	if err != nil {
+		return "", ""
+	}
+	return u.Host, u.Path
+}
+
+func cleanUrlFromRequest(r *http.Request) (string, string) {
 	host := r.Header.Get("x-forwarded-host")
 	if host == "" {
 		host = r.URL.Host
 
 var ErrAnalyticsDisabled = errors.New("owner does not have site analytics enabled")
 
-func AnalyticsVisitFromRequest(r *http.Request, dbpool db.DB, userID string, secret string) (*db.AnalyticsVisits, error) {
-	if !dbpool.HasFeatureForUser(userID, "analytics") {
-		return nil, ErrAnalyticsDisabled
+func AnalyticsVisitFromVisit(visit *db.AnalyticsVisits, dbpool db.DB, secret string) error {
+	if !dbpool.HasFeatureForUser(visit.UserID, "analytics") {
+		return ErrAnalyticsDisabled
 	}
 
-	err := trackableRequest(r)
+	err := trackableUserAgent(visit.UserAgent)
 	if err != nil {
-		return nil, err
+		return err
+	}
+
+	ipAddress, err := cleanIpAddress(visit.IpAddress)
+	if err != nil {
+		return err
+	}
+	visit.IpAddress = HmacString(secret, ipAddress)
+	_, path := cleanUrl(visit.Path)
+	visit.Path = path
+
+	referer, err := cleanReferer(visit.Referer)
+	if err != nil {
+		return err
 	}
+	visit.Referer = referer
+	visit.UserAgent = cleanUserAgent(visit.UserAgent)
 
+	return nil
+}
+
+func ipFromRequest(r *http.Request) string {
 	// https://caddyserver.com/docs/caddyfile/directives/reverse_proxy#defaults
 	ipOrig := r.Header.Get("x-forwarded-for")
 	if ipOrig == "" {
 			ipOrig = sshCtx.RemoteAddr().String()
 		}
 	}
-	ipAddress, err := cleanIpAddress(ipOrig)
-	if err != nil {
-		return nil, err
+
+	return ipOrig
+}
+
+func AnalyticsVisitFromRequest(r *http.Request, dbpool db.DB, userID string) (*db.AnalyticsVisits, error) {
+	if !dbpool.HasFeatureForUser(userID, "analytics") {
+		return nil, ErrAnalyticsDisabled
 	}
-	host, path := cleanUrl(r)
 
-	referer, err := cleanReferer(r.Referer())
+	err := trackableRequest(r)
 	if err != nil {
 		return nil, err
 	}
 
+	ipAddress := ipFromRequest(r)
+	host, path := cleanUrlFromRequest(r)
+
 	return &db.AnalyticsVisits{
 		UserID:    userID,
 		Host:      host,
 		Path:      path,
-		IpAddress: HmacString(secret, ipAddress),
-		UserAgent: cleanUserAgent(r.UserAgent()),
-		Referer:   referer,
+		IpAddress: ipAddress,
+		UserAgent: r.UserAgent(),
+		Referer:   r.Referer(),
 		Status:    http.StatusOK,
 	}, nil
 }

old

new

 type ConfigSite struct {
 	Debug              bool
 	SendgridKey        string
-	Secret             string
 	Domain             string
 	Port               string
 	PortOverride       string

old

new

+ALTER TABLE analytics_visits ADD COLUMN namespace varchar(256);