dashboard / tuns / feat: metric drain #34 rss

accepted · opened on 2024-11-13T02:18:51Z by erock
Help 
# add changes to patch request
git format-patch main --stdout | ssh pr.pico.sh pr add 34
# add review to patch request
git format-patch main --stdout | ssh pr.pico.sh pr add --review 34
# remove patchset
ssh pr.pico.sh ps rm ps-x
# checkout all patches
ssh pr.pico.sh pr print 34 | git am -3
# print a diff between the last two patches in a patch request
ssh pr.pico.sh pr diff 34
# accept PR
ssh pr.pico.sh pr accept 34
# close PR
ssh pr.pico.sh pr close 34

Logs

erock created pr with ps-69 on 2024-11-13T02:18:51Z
erock added ps-70 on 2024-11-14T03:06:46Z
erock added ps-71 on 2024-11-14T03:22:31Z
erock changed status on 2024-11-23T03:33:38Z {"status":"accepted"}

Patchsets

ps-69 by erock on 2024-11-13T02:18:51Z
Range Diff ↕
1: 4bd8050 = 1: 4bd8050 feat: metric drain
-: ------- > 2: ea49338 chore: add userId to metric drain visit
ps-70 by erock on 2024-11-14T03:06:46Z
Range Diff ↕
1: 4bd8050 = 1: 4bd8050 feat: metric drain
2: ea49338 = 2: ea49338 chore: add userId to metric drain visit
-: ------- > 3: 343f29e chore: update pubsub
ps-71 by erock on 2024-11-14T03:22:31Z

Patchset ps-71

feat: metric drain

Eric Bower
2024-11-13T02:16:16Z
Makefile
+4 -0
cmd/sish.go
+7 -2
go.mod
+2 -0
httpmuxer/httpmuxer.go
+28 -0
sshmuxer/sshmuxer.go
+13 -0
utils/conn.go
+10 -0
utils/state.go
+1 -0
utils/utils.go
+6 -0

chore: add userId to metric drain visit

Eric Bower
2024-11-14T03:05:56Z
httpmuxer/httpmuxer.go
+24 -19

chore: update pubsub

Eric Bower
2024-11-14T03:21:06Z
go.mod
+1 -3
go.sum
+2 -2
Back to top

feat: metric drain

This change will enable http remote forwards to have site analytics
enabled automatically.
Makefile link
+4 -0 
 1diff --git a/Makefile b/Makefile
 2index 8b36002..31e617e 100644
 3--- a/Makefile
 4+++ b/Makefile
 5@@ -19,3 +19,7 @@ docs-prod: ssg
 6 dev:
 7 	go run main.go --http-address localhost:3000 --domain testing.ssi.sh
 8 .PHONY: dev
 9+
10+fmt:
11+	go fmt ./...
12+.PHONY: fmt
cmd/sish.go link
+7 -2 
 1diff --git a/cmd/sish.go b/cmd/sish.go
 2index 159f02e..bba3f45 100644
 3--- a/cmd/sish.go
 4+++ b/cmd/sish.go
 5@@ -17,6 +17,7 @@ import (
 6 	"github.com/spf13/viper"
 7 	"gopkg.in/natefinch/lumberjack.v2"
 8 
 9+	"github.com/picosh/pubsub"
10 	pipeLogger "github.com/picosh/pubsub/log"
11 	picoUtils "github.com/picosh/utils"
12 )
13@@ -206,17 +207,21 @@ func initConfig() {
14 		slog.NewTextHandler(multiWriter, &slog.HandlerOptions{AddSource: true, Level: logLevel}),
15 	)
16 
17-	realLogger, err := pipeLogger.SendLogRegister(logger, &pipeLogger.PubSubConnectionInfo{
18+	info := &pubsub.RemoteClientInfo{
19 		RemoteHost:     picoUtils.GetEnv("PICO_PIPE_ENDPOINT", "send.pico.sh:22"),
20 		KeyLocation:    picoUtils.GetEnv("PICO_PIPE_KEY", "ssh_data/term_info_ed25519"),
21 		KeyPassphrase:  picoUtils.GetEnv("PICO_PIPE_PASSPHRASE", ""),
22 		RemoteHostname: picoUtils.GetEnv("PICO_PIPE_REMOTE_HOST", "send.pico.sh"),
23 		RemoteUser:     picoUtils.GetEnv("PICO_PIPE_USER", "pico"),
24-	}, 100)
25+	}
26+	realLogger, err := pipeLogger.SendLogRegister(logger, info, 100)
27 	if err != nil {
28 		slog.Error("unable to set real logger", slog.Any("error", err))
29 	}
30 
31+	metricDrain := pubsub.NewRemoteClientWriter(info, logger, 100)
32+	go metricDrain.KeepAlive("pub metric-drain -b=false")
33+
34 	trueLogger := logger
35 
36 	if realLogger != nil {
go.mod link
+2 -0 
 1diff --git a/go.mod b/go.mod
 2index 32259c2..043199f 100644
 3--- a/go.mod
 4+++ b/go.mod
 5@@ -2,6 +2,8 @@ module github.com/antoniomika/sish
 6 
 7 go 1.23.1
 8 
 9+replace github.com/picosh/pubsub => /home/erock/dev/pico/pubsub
10+
11 require (
12 	github.com/ScaleFT/sshkeys v1.2.0
13 	github.com/antoniomika/multilistener v0.0.0-20240307222635-f0dc097d8acc
httpmuxer/httpmuxer.go link
+28 -0 
 1diff --git a/httpmuxer/httpmuxer.go b/httpmuxer/httpmuxer.go
 2index 7ab9094..1888123 100644
 3--- a/httpmuxer/httpmuxer.go
 4+++ b/httpmuxer/httpmuxer.go
 5@@ -31,6 +31,16 @@ import (
 6 	"github.com/gin-gonic/gin"
 7 )
 8 
 9+type MetricDrainVisit struct {
10+	UserID    string `json:"user_id"`
11+	Host      string `json:"host"`
12+	Path      string `json:"path"`
13+	IpAddress string `json:"ip_adress"`
14+	UserAgent string `json:"user_agent"`
15+	Referer   string `json:"referer"`
16+	Status    int    `json:"status"`
17+}
18+
19 // Start initializes the HTTP service.
20 func Start(state *utils.State) {
21 	releaseMode := gin.ReleaseMode
22@@ -106,6 +116,24 @@ func Start(state *utils.State) {
23 			param.ErrorMessage,
24 		)
25 
26+		visit := MetricDrainVisit{
27+			UserID:    "",
28+			Host:      param.Request.Host,
29+			IpAddress: param.ClientIP,
30+			Status:    param.StatusCode,
31+			Path:      originalURI,
32+			UserAgent: param.Request.UserAgent(),
33+			Referer:   param.Request.Referer(),
34+		}
35+		data, err := json.Marshal(visit)
36+		if err != nil {
37+			slog.Error("could not json marshall visit record", "err", err)
38+		}
39+		_, err = state.MetricDrain.Write(data)
40+		if err != nil {
41+			slog.Error("could not send to visit to metric-drain", "err", err)
42+		}
43+
44 		slog.Info(logLine)
45 
46 		if viper.GetBool("log-to-client") && param.Keys["httpHolder"] != nil {
sshmuxer/sshmuxer.go link
+13 -0 
 1diff --git a/sshmuxer/sshmuxer.go b/sshmuxer/sshmuxer.go
 2index e200bb7..1a6c014 100644
 3--- a/sshmuxer/sshmuxer.go
 4+++ b/sshmuxer/sshmuxer.go
 5@@ -16,6 +16,8 @@ import (
 6 	"github.com/antoniomika/sish/httpmuxer"
 7 	"github.com/antoniomika/sish/utils"
 8 	"github.com/antoniomika/syncmap"
 9+	"github.com/picosh/pubsub"
10+	picoUtils "github.com/picosh/utils"
11 	"github.com/pires/go-proxyproto"
12 	"github.com/spf13/viper"
13 	"golang.org/x/crypto/ssh"
14@@ -70,10 +72,21 @@ func Start() {
15 
16 	utils.WatchKeys()
17 
18+	info := &pubsub.RemoteClientInfo{
19+		RemoteHost:     picoUtils.GetEnv("PICO_PIPE_ENDPOINT", "send.pico.sh:22"),
20+		KeyLocation:    picoUtils.GetEnv("PICO_PIPE_KEY", "ssh_data/term_info_ed25519"),
21+		KeyPassphrase:  picoUtils.GetEnv("PICO_PIPE_PASSPHRASE", ""),
22+		RemoteHostname: picoUtils.GetEnv("PICO_PIPE_REMOTE_HOST", "send.pico.sh"),
23+		RemoteUser:     picoUtils.GetEnv("PICO_PIPE_USER", "pico"),
24+	}
25+	metricDrain := pubsub.NewRemoteClientWriter(info, slog.Default(), 100)
26+	go metricDrain.KeepAlive("pub metric-drain -b=false")
27+
28 	state := utils.NewState()
29 	state.Ports.HTTPPort = httpPort
30 	state.Ports.HTTPSPort = httpsPort
31 	state.Ports.SSHPort = sshPort
32+	state.MetricDrain = metricDrain
33 
34 	state.Console.State = state
35
utils/conn.go link
+10 -0 
 1diff --git a/utils/conn.go b/utils/conn.go
 2index 4ce0ebc..57ef9e6 100644
 3--- a/utils/conn.go
 4+++ b/utils/conn.go
 5@@ -99,6 +99,16 @@ func (s *SSHConnection) User() string {
 6 	return user
 7 }
 8 
 9+// User returns the user of the session, either using the extensions user or the passed user.
10+func (s *SSHConnection) UserId() string {
11+	user, ok := s.SSHConn.Permissions.Extensions["userId"]
12+	if !ok {
13+		user = ""
14+	}
15+
16+	return user
17+}
18+
19 // TeeConn represents a simple net.Conn interface for SNI Processing.
20 type TeeConn struct {
21 	Conn     net.Conn
utils/state.go link
+1 -0 
 1diff --git a/utils/state.go b/utils/state.go
 2index 594c299..f27e201 100644
 3--- a/utils/state.go
 4+++ b/utils/state.go
 5@@ -231,6 +231,7 @@ type State struct {
 6 	ActiveConnections prometheus.GaugeFunc
 7 	ActiveTunnels     *prometheus.GaugeVec
 8 	ConnectionTime    *prometheus.GaugeVec
 9+	MetricDrain       io.Writer
10 }
11 
12 // NewState returns a new State struct.
utils/utils.go link
+6 -0 
 1diff --git a/utils/utils.go b/utils/utils.go
 2index 013c13c..aa3c403 100644
 3--- a/utils/utils.go
 4+++ b/utils/utils.go
 5@@ -597,7 +597,13 @@ func checkAuthenticationKeyRequest(authUrl string, authKey []byte, addr net.Addr
 6 			return false, extensionsInfo, nil
 7 		}
 8 
 9+		userId, ok := userData["id"].(string)
10+		if !ok {
11+			return false, extensionsInfo, nil
12+		}
13+
14 		extensionsInfo["user"] = userName
15+		extensionsInfo["userId"] = userId
16 	}
17 
18 	return true, extensionsInfo, nil

chore: add userId to metric drain visit

chore: add user scope to http request log line
httpmuxer/httpmuxer.go link
+24 -19 
 1diff --git a/httpmuxer/httpmuxer.go b/httpmuxer/httpmuxer.go
 2index 1888123..8bf5317 100644
 3--- a/httpmuxer/httpmuxer.go
 4+++ b/httpmuxer/httpmuxer.go
 5@@ -116,25 +116,30 @@ func Start(state *utils.State) {
 6 			param.ErrorMessage,
 7 		)
 8 
 9-		visit := MetricDrainVisit{
10-			UserID:    "",
11-			Host:      param.Request.Host,
12-			IpAddress: param.ClientIP,
13-			Status:    param.StatusCode,
14-			Path:      originalURI,
15-			UserAgent: param.Request.UserAgent(),
16-			Referer:   param.Request.Referer(),
17-		}
18-		data, err := json.Marshal(visit)
19-		if err != nil {
20-			slog.Error("could not json marshall visit record", "err", err)
21-		}
22-		_, err = state.MetricDrain.Write(data)
23-		if err != nil {
24-			slog.Error("could not send to visit to metric-drain", "err", err)
25-		}
26-
27-		slog.Info(logLine)
28+		state.SSHConnections.Range(func(I string, conn *utils.SSHConnection) bool {
29+			user := conn.User()
30+			userId := conn.UserId()
31+			slog.Info(logLine, "user", user, "userId", userId)
32+
33+			visit := MetricDrainVisit{
34+				UserID:    userId,
35+				Host:      param.Request.Host,
36+				IpAddress: param.ClientIP,
37+				Status:    param.StatusCode,
38+				Path:      originalURI,
39+				UserAgent: param.Request.UserAgent(),
40+				Referer:   param.Request.Referer(),
41+			}
42+			data, err := json.Marshal(visit)
43+			if err != nil {
44+				slog.Error("could not json marshall visit record", "err", err)
45+			}
46+			_, err = state.MetricDrain.Write(data)
47+			if err != nil {
48+				slog.Error("could not send to visit to metric-drain", "err", err)
49+			}
50+			return true
51+		})
52 
53 		if viper.GetBool("log-to-client") && param.Keys["httpHolder"] != nil {
54 			currentListener := param.Keys["httpHolder"].(*utils.HTTPHolder)

chore: update pubsub

go.mod link
+1 -3 
 1diff --git a/go.mod b/go.mod
 2index 043199f..3fad528 100644
 3--- a/go.mod
 4+++ b/go.mod
 5@@ -2,8 +2,6 @@ module github.com/antoniomika/sish
 6 
 7 go 1.23.1
 8 
 9-replace github.com/picosh/pubsub => /home/erock/dev/pico/pubsub
10-
11 require (
12 	github.com/ScaleFT/sshkeys v1.2.0
13 	github.com/antoniomika/multilistener v0.0.0-20240307222635-f0dc097d8acc
14@@ -16,7 +14,7 @@ require (
15 	github.com/logrusorgru/aurora v2.0.3+incompatible
16 	github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a
17 	github.com/picosh/pdocs v0.0.0-20240626184614-7aff67648d6e
18-	github.com/picosh/pubsub v0.0.0-20241008010300-a63fd95dc8ed
19+	github.com/picosh/pubsub v0.0.0-20241114025640-35db438302b4
20 	github.com/picosh/utils v0.0.0-20241008004349-f48b50af554b
21 	github.com/pires/go-proxyproto v0.7.0
22 	github.com/prometheus/client_golang v1.20.2
go.sum link
+2 -2 
 1diff --git a/go.sum b/go.sum
 2index fd2d443..611100f 100644
 3--- a/go.sum
 4+++ b/go.sum
 5@@ -136,8 +136,8 @@ github.com/phuslu/iploc v1.0.20240731 h1:4fLlMTe1bGrKfraWclGGC3WqndKzDv7xpKgVRvX
 6 github.com/phuslu/iploc v1.0.20240731/go.mod h1:VZqAWoi2A80YPvfk1AizLGHavNIG9nhBC8d87D/SeVs=
 7 github.com/picosh/pdocs v0.0.0-20240626184614-7aff67648d6e h1:SdfU2G7pMYi1pZd4khx/prmgfF5WZ0pRROhCpPMq0H4=
 8 github.com/picosh/pdocs v0.0.0-20240626184614-7aff67648d6e/go.mod h1:KXO3Z0EVdA811AX6mlK4lwFDT+KgmegRVrEmZU5uLXU=
 9-github.com/picosh/pubsub v0.0.0-20241008010300-a63fd95dc8ed h1:aBJeQoLvq/V3hX6bgWjuuTmGzgbPNYuuwaCWU4aSJcU=
10-github.com/picosh/pubsub v0.0.0-20241008010300-a63fd95dc8ed/go.mod h1:ajolgob5MxlHdp5HllF7u3rTlCgER4InqfP7M/xl6HQ=
11+github.com/picosh/pubsub v0.0.0-20241114025640-35db438302b4 h1:pITSRXb9NDGdC6AmuS3JE+8Ek4/pUG7tXJPP3cOaqf4=
12+github.com/picosh/pubsub v0.0.0-20241114025640-35db438302b4/go.mod h1:m6ZZpg+lZB3XTIKlbSqQgi4NrBPtARv23b8vGYDoCo4=
13 github.com/picosh/utils v0.0.0-20241008004349-f48b50af554b h1:PvWk8Y7JhC1bK4Ns7FUFfcvi+BGZ+K07wTA2VDTmfDQ=
14 github.com/picosh/utils v0.0.0-20241008004349-f48b50af554b/go.mod h1:ftrp1FjbKK/mFnBAYGymA1QEtPlkA0+lWkPI5h0HKt4=
15 github.com/pires/go-proxyproto v0.7.0 h1:IukmRewDQFWC7kfnb66CSomk2q/seBuilHBYFwyq0Hs=