pico

reactor(metric-drain): use caddy json format open rss

created pr with 35.1 on 2024-11-15T15:03:31Z · by c8ef7d19

added 35.2 on 2024-11-23T03:34:44Z · by c8ef7d19


                      1:  77aaa29 ! 1:  8d56535 reactor(metric-drain): use caddy json format


                      -:  ------- > 2:  a336041 wip


                      -:  ------- > 3:  7ae45b3 chore: wrap


                      -:  ------- > 4:  bfa5c4f done

added 35.3 on 2024-11-27T20:17:20Z · by c8ef7d19


                      1:  8d56535 < -:  ------- reactor(metric-drain): use caddy json format


                      -:  ------- > 1:  c7eeb12 reactor(metric-drain): use caddy access logs


                      2:  a336041 < -:  ------- wip


                      3:  7ae45b3 < -:  ------- chore: wrap


                      4:  bfa5c4f < -:  ------- done

added 35.4 on 2024-11-27T20:18:31Z · by c8ef7d19


                      1:  c7eeb12 ! 1:  4e0839a reactor(metric-drain): use caddy access logs

changed status to accepted on 2024-11-28T03:03:53Z · by c8ef7d19

cmds

checkout latest patchset:

ssh pr.pico.sh print 35 | git am -3

checkout any patchset in a patch request:

ssh pr.pico.sh print 35.[rev] | git am -3

add changes to patch request:

git format-patch main --stdout | ssh pr.pico.sh pr add 35

set PR to open (enables RSS notifications):

ssh pr.pico.sh pr open 35

set PR to draft (stops RSS notifications):

ssh pr.pico.sh pr draft 35

Patchset `35.1` on 2024-11-15T15:03:31Z · commit 77aaa29

→ reactor(metric-drain): use caddy json format

Eric Bower 2024-11-15T15:02:24Z

reactor(metric-drain): use caddy json format

Eric Bower 2024-11-15T15:02:24Z

Semantic diff summary

5 added, 4 modified, 0 signature changed, 2 removed across 4 analyzed files (1 file skipped: unsupported file type)

auth/api.go

type_declaration AccessLogReq added
type_declaration CaddyAccessLog added
function_declaration deserializeCaddyAccessLog added
function_declaration metricDrainSub modified

pgs/tunnel.go

function_declaration createHttpHandler modified

pgs/web.go

function_declaration checkHandler modified
function_declaration ServeAsset modified
type_declaration SubdomainProps removed
function_declaration getProjectFromSubdomain removed

shared/api.go

type_declaration SubdomainProps added
function_declaration GetProjectFromSubdomain added

+92 -4 auth/api.go #

......@@ -642,6 +642,89 @@ func handler(routes []shared.Route, client *Client) http.HandlerFunc {
	}
}

type AccessLogReq struct {
	RemoteIP   string `json:"remote_ip"`
	RemotePort string `json:"remote_port"`
	ClientIP   string `json:"client_ip"`
	Method     string `json:"method"`
	Host       string `json:"host"`
	Uri        string `json:"uri"`
	Headers    struct {
		UserAgent string `json:"User-Agent"`
		Referer   string `json:"Referer"`
	} `json:"headers"`
	Tls struct {
		ServerName string `json:"server_name"`
	} `json:"tls"`
}

type CaddyAccessLog struct {
	Request AccessLogReq `json:"request"`
	Status  int          `json:"status"`
}

func deserializeCaddyAccessLog(dbpool db.DB, access *CaddyAccessLog) (*db.AnalyticsVisits, error) {
	spaceRaw := strings.SplitN(access.Request.Tls.ServerName, ".", 2)
	space := spaceRaw[0]
	host := access.Request.Host
	path := access.Request.Uri
	subdomain := ""

	// grab subdomain based on host
	if strings.HasSuffix(host, "tuns.sh") {
		subdomain = strings.TrimSuffix(host, ".tuns.sh")
	} else if strings.HasSuffix(host, "pgs.sh") {
		subdomain = strings.TrimSuffix(host, ".pgs.sh")
	} else if strings.HasSuffix(host, "prose.sh") {
		subdomain = strings.TrimSuffix(host, ".prose.sh")
	} else {
		subdomain = shared.GetCustomDomain(host, space)
	}

	// get user and namespace details from subdomain
	props, err := shared.GetProjectFromSubdomain(subdomain)
	if err != nil {
		return nil, err
	}
	// get user ID
	user, err := dbpool.FindUserForName(props.Username)
	if err != nil {
		return nil, err
	}

	projectID := ""
	postID := ""
	if space == "pgs" { // figure out project ID
		project, err := dbpool.FindProjectByName(user.ID, props.ProjectName)
		if err != nil {
			return nil, err
		}
		projectID = project.ID
	} else if space == "prose" { // figure out post ID
		if path == "" || path == "/" {
		} else {
			post, err := dbpool.FindPostWithSlug(path, user.ID, space)
			if err != nil {
				return nil, err
			}
			postID = post.ID
		}
	}

	return &db.AnalyticsVisits{
		UserID:    user.ID,
		ProjectID: projectID,
		PostID:    postID,
		Namespace: space,
		Host:      host,
		Path:      path,
		IpAddress: access.Request.ClientIP,
		UserAgent: access.Request.Headers.UserAgent,
		Referer:   access.Request.Headers.Referer, // TODO: I don't see referer in the access log
		Status:    access.Status,
	}, nil
}

func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secret string) {
	conn := shared.NewPicoPipeClient()
	stdoutPipe, err := pubsub.RemoteSub("sub metric-drain -k", ctx, conn)

......@@ -654,14 +737,19 @@ func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secr
	scanner := bufio.NewScanner(stdoutPipe)
	for scanner.Scan() {
		line := scanner.Text()
		visit := db.AnalyticsVisits{}
		err := json.Unmarshal([]byte(line), &visit)
		accessLog := CaddyAccessLog{}
		err := json.Unmarshal([]byte(line), &accessLog)
		if err != nil {
			logger.Error("json unmarshal", "err", err)
			continue
		}

		err = shared.AnalyticsVisitFromVisit(&visit, dbpool, secret)
		visit, err := deserializeCaddyAccessLog(dbpool, &accessLog)
		if err != nil {
			logger.Error("cannot deserialize access log", "err", err)
			continue
		}
		err = shared.AnalyticsVisitFromVisit(visit, dbpool, secret)
		if err != nil {
			if !errors.Is(err, shared.ErrAnalyticsDisabled) {
				logger.Info("could not record analytics visit", "reason", err)

......@@ -669,7 +757,7 @@ func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secr
		}

		logger.Info("inserting visit", "visit", visit)
		err = dbpool.InsertVisit(&visit)
		err = dbpool.InsertVisit(visit)
		if err != nil {
			logger.Error("could not insert visit record", "err", err)
		}

+40 -0 caddy.json #

......@@ -0,0 +1,40 @@
{
  "level": "info",
  "ts": 1731644477.313701,
  "logger": "http.log.access",
  "msg": "handled request",
  "request": {
    "remote_ip": "127.0.0.1",
    "remote_port": "40400",
    "client_ip": "127.0.0.1",
    "proto": "HTTP/2.0",
    "method": "GET",
    "host": "pgs.sh",
    "uri": "/",
    "headers": { "User-Agent": ["Blackbox Exporter/0.24.0"] },
    "tls": {
      "resumed": false,
      "version": 772,
      "cipher_suite": 4865,
      "proto": "h2",
      "server_name": "pgs.sh"
    }
  },
  "bytes_read": 0,
  "user_id": "",
  "duration": 0.001207084,
  "size": 3718,
  "status": 200,
  "resp_headers": {
    "Referrer-Policy": ["no-referrer-when-downgrade"],
    "Strict-Transport-Security": ["max-age=31536000;"],
    "X-Content-Type-Options": ["nosniff"],
    "X-Frame-Options": ["DENY"],
    "Server": ["Caddy"],
    "Alt-Svc": ["h3=\":443\"; ma=2592000"],
    "Date": ["Fri, 15 Nov 2024 04:21:17 GMT"],
    "Content-Type": ["text/html; charset=utf-8"],
    "X-Xss-Protection": ["1; mode=block"],
    "Permissions-Policy": ["interest-cohort=()"]
  }
}

+1 -1 pgs/tunnel.go #

......@@ -51,7 +51,7 @@ func createHttpHandler(apiConfig *shared.ApiConfig) CtxHttpBridge {
			"pubkey", pubkeyStr,
		)

		props, err := getProjectFromSubdomain(subdomain)
		props, err := shared.GetProjectFromSubdomain(subdomain)
		if err != nil {
			log.Error(err.Error())
			return http.HandlerFunc(shared.UnauthorizedHandler)

+2 -19 pgs/web.go #

......@@ -174,7 +174,7 @@ func (web *WebRouter) checkHandler(w http.ResponseWriter, r *http.Request) {

		if !strings.Contains(hostDomain, appDomain) {
			subdomain := shared.GetCustomDomain(hostDomain, cfg.Space)
			props, err := getProjectFromSubdomain(subdomain)
			props, err := shared.GetProjectFromSubdomain(subdomain)
			if err != nil {
				logger.Error(
					"could not get project from subdomain",

......@@ -330,7 +330,7 @@ func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, fro
		"host", r.Host,
	)

	props, err := getProjectFromSubdomain(subdomain)
	props, err := shared.GetProjectFromSubdomain(subdomain)
	if err != nil {
		logger.Info(
			"could not determine project from subdomain",

......@@ -447,20 +447,3 @@ func (web *WebRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ctx = context.WithValue(ctx, shared.CtxSubdomainKey{}, subdomain)
	router.ServeHTTP(w, r.WithContext(ctx))
}

type SubdomainProps struct {
	ProjectName string
	Username    string
}

func getProjectFromSubdomain(subdomain string) (*SubdomainProps, error) {
	props := &SubdomainProps{}
	strs := strings.SplitN(subdomain, "-", 2)
	props.Username = strs[0]
	if len(strs) == 2 {
		props.ProjectName = strs[1]
	} else {
		props.ProjectName = props.Username
	}
	return props, nil
}

+17 -0 shared/api.go #

......@@ -13,6 +13,23 @@ import (
	"github.com/picosh/utils"
)

type SubdomainProps struct {
	ProjectName string
	Username    string
}

func GetProjectFromSubdomain(subdomain string) (*SubdomainProps, error) {
	props := &SubdomainProps{}
	strs := strings.SplitN(subdomain, "-", 2)
	props.Username = strs[0]
	if len(strs) == 2 {
		props.ProjectName = strs[1]
	} else {
		props.ProjectName = props.Username
	}
	return props, nil
}

func CorsHeaders(headers http.Header) {
	headers.Add("Access-Control-Allow-Origin", "*")
	headers.Add("Vary", "Origin")

...	...		@@ -642,6 +642,89 @@ func handler(routes []shared.Route, client *Client) http.HandlerFunc {
642	642		}
643	643		}
644	644
	645	+	type AccessLogReq struct {
	646	+	RemoteIP string `json:"remote_ip"`
	647	+	RemotePort string `json:"remote_port"`
	648	+	ClientIP string `json:"client_ip"`
	649	+	Method string `json:"method"`
	650	+	Host string `json:"host"`
	651	+	Uri string `json:"uri"`
	652	+	Headers struct {
	653	+	UserAgent string `json:"User-Agent"`
	654	+	Referer string `json:"Referer"`
	655	+	} `json:"headers"`
	656	+	Tls struct {
	657	+	ServerName string `json:"server_name"`
	658	+	} `json:"tls"`
	659	+	}
	660	+
	661	+	type CaddyAccessLog struct {
	662	+	Request AccessLogReq `json:"request"`
	663	+	Status int `json:"status"`
	664	+	}
	665	+
	666	+	func deserializeCaddyAccessLog(dbpool db.DB, access CaddyAccessLog) (db.AnalyticsVisits, error) {
	667	+	spaceRaw := strings.SplitN(access.Request.Tls.ServerName, ".", 2)
	668	+	space := spaceRaw[0]
	669	+	host := access.Request.Host
	670	+	path := access.Request.Uri
	671	+	subdomain := ""
	672	+
	673	+	// grab subdomain based on host
	674	+	if strings.HasSuffix(host, "tuns.sh") {
	675	+	subdomain = strings.TrimSuffix(host, ".tuns.sh")
	676	+	} else if strings.HasSuffix(host, "pgs.sh") {
	677	+	subdomain = strings.TrimSuffix(host, ".pgs.sh")
	678	+	} else if strings.HasSuffix(host, "prose.sh") {
	679	+	subdomain = strings.TrimSuffix(host, ".prose.sh")
	680	+	} else {
	681	+	subdomain = shared.GetCustomDomain(host, space)
	682	+	}
	683	+
	684	+	// get user and namespace details from subdomain
	685	+	props, err := shared.GetProjectFromSubdomain(subdomain)
	686	+	if err != nil {
	687	+	return nil, err
	688	+	}
	689	+	// get user ID
	690	+	user, err := dbpool.FindUserForName(props.Username)
	691	+	if err != nil {
	692	+	return nil, err
	693	+	}
	694	+
	695	+	projectID := ""
	696	+	postID := ""
	697	+	if space == "pgs" { // figure out project ID
	698	+	project, err := dbpool.FindProjectByName(user.ID, props.ProjectName)
	699	+	if err != nil {
	700	+	return nil, err
	701	+	}
	702	+	projectID = project.ID
	703	+	} else if space == "prose" { // figure out post ID
	704	+	if path == "" \|\| path == "/" {
	705	+	} else {
	706	+	post, err := dbpool.FindPostWithSlug(path, user.ID, space)
	707	+	if err != nil {
	708	+	return nil, err
	709	+	}
	710	+	postID = post.ID
	711	+	}
	712	+	}
	713	+
	714	+	return &db.AnalyticsVisits{
	715	+	UserID: user.ID,
	716	+	ProjectID: projectID,
	717	+	PostID: postID,
	718	+	Namespace: space,
	719	+	Host: host,
	720	+	Path: path,
	721	+	IpAddress: access.Request.ClientIP,
	722	+	UserAgent: access.Request.Headers.UserAgent,
	723	+	Referer: access.Request.Headers.Referer, // TODO: I don't see referer in the access log
	724	+	Status: access.Status,
	725	+	}, nil
	726	+	}
	727	+
645	728		func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secret string) {
646	729		conn := shared.NewPicoPipeClient()
647	730		stdoutPipe, err := pubsub.RemoteSub("sub metric-drain -k", ctx, conn)

...	...		@@ -654,14 +737,19 @@ func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secr
654	737		scanner := bufio.NewScanner(stdoutPipe)
655	738		for scanner.Scan() {
656	739		line := scanner.Text()
657		-	visit := db.AnalyticsVisits{}
658		-	err := json.Unmarshal([]byte(line), &visit)
	740	+	accessLog := CaddyAccessLog{}
	741	+	err := json.Unmarshal([]byte(line), &accessLog)
659	742		if err != nil {
660	743		logger.Error("json unmarshal", "err", err)
661	744		continue
662	745		}
663	746
664		-	err = shared.AnalyticsVisitFromVisit(&visit, dbpool, secret)
	747	+	visit, err := deserializeCaddyAccessLog(dbpool, &accessLog)
	748	+	if err != nil {
	749	+	logger.Error("cannot deserialize access log", "err", err)
	750	+	continue
	751	+	}
	752	+	err = shared.AnalyticsVisitFromVisit(visit, dbpool, secret)
665	753		if err != nil {
666	754		if !errors.Is(err, shared.ErrAnalyticsDisabled) {
667	755		logger.Info("could not record analytics visit", "reason", err)

...	...		@@ -669,7 +757,7 @@ func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secr
669	757		}
670	758
671	759		logger.Info("inserting visit", "visit", visit)
672		-	err = dbpool.InsertVisit(&visit)
	760	+	err = dbpool.InsertVisit(visit)
673	761		if err != nil {
674	762		logger.Error("could not insert visit record", "err", err)
675	763		}

...	...		@@ -447,20 +447,3 @@ func (web WebRouter) ServeHTTP(w http.ResponseWriter, r http.Request) {
447	447		ctx = context.WithValue(ctx, shared.CtxSubdomainKey{}, subdomain)
448	448		router.ServeHTTP(w, r.WithContext(ctx))
449	449		}
450		-
451		-	type SubdomainProps struct {
452		-	ProjectName string
453		-	Username string
454		-	}
455		-
456		-	func getProjectFromSubdomain(subdomain string) (*SubdomainProps, error) {
457		-	props := &SubdomainProps{}
458		-	strs := strings.SplitN(subdomain, "-", 2)
459		-	props.Username = strs[0]
460		-	if len(strs) == 2 {
461		-	props.ProjectName = strs[1]
462		-	} else {
463		-	props.ProjectName = props.Username
464		-	}
465		-	return props, nil
466		-	}

...	...		@@ -0,0 +1,40 @@
	1	+	{
	2	+	"level": "info",
	3	+	"ts": 1731644477.313701,
	4	+	"logger": "http.log.access",
	5	+	"msg": "handled request",
	6	+	"request": {
	7	+	"remote_ip": "127.0.0.1",
	8	+	"remote_port": "40400",
	9	+	"client_ip": "127.0.0.1",
	10	+	"proto": "HTTP/2.0",
	11	+	"method": "GET",
	12	+	"host": "pgs.sh",
	13	+	"uri": "/",
	14	+	"headers": { "User-Agent": ["Blackbox Exporter/0.24.0"] },
	15	+	"tls": {
	16	+	"resumed": false,
	17	+	"version": 772,
	18	+	"cipher_suite": 4865,
	19	+	"proto": "h2",
	20	+	"server_name": "pgs.sh"
	21	+	}
	22	+	},
	23	+	"bytes_read": 0,
	24	+	"user_id": "",
	25	+	"duration": 0.001207084,
	26	+	"size": 3718,
	27	+	"status": 200,
	28	+	"resp_headers": {
	29	+	"Referrer-Policy": ["no-referrer-when-downgrade"],
	30	+	"Strict-Transport-Security": ["max-age=31536000;"],
	31	+	"X-Content-Type-Options": ["nosniff"],
	32	+	"X-Frame-Options": ["DENY"],
	33	+	"Server": ["Caddy"],
	34	+	"Alt-Svc": ["h3=\":443\"; ma=2592000"],
	35	+	"Date": ["Fri, 15 Nov 2024 04:21:17 GMT"],
	36	+	"Content-Type": ["text/html; charset=utf-8"],
	37	+	"X-Xss-Protection": ["1; mode=block"],
	38	+	"Permissions-Policy": ["interest-cohort=()"]
	39	+	}
	40	+	}

...	...		@@ -51,7 +51,7 @@ func createHttpHandler(apiConfig *shared.ApiConfig) CtxHttpBridge {
51	51		"pubkey", pubkeyStr,
52	52		)
53	53
54		-	props, err := getProjectFromSubdomain(subdomain)
	54	+	props, err := shared.GetProjectFromSubdomain(subdomain)
55	55		if err != nil {
56	56		log.Error(err.Error())
57	57		return http.HandlerFunc(shared.UnauthorizedHandler)

...	...		@@ -174,7 +174,7 @@ func (web WebRouter) checkHandler(w http.ResponseWriter, r http.Request) {
174	174
175	175		if !strings.Contains(hostDomain, appDomain) {
176	176		subdomain := shared.GetCustomDomain(hostDomain, cfg.Space)
177		-	props, err := getProjectFromSubdomain(subdomain)
	177	+	props, err := shared.GetProjectFromSubdomain(subdomain)
178	178		if err != nil {
179	179		logger.Error(
180	180		"could not get project from subdomain",

...	...		@@ -330,7 +330,7 @@ func (web WebRouter) ServeAsset(fname string, opts storage.ImgProcessOpts, fro
330	330		"host", r.Host,
331	331		)
332	332
333		-	props, err := getProjectFromSubdomain(subdomain)
	333	+	props, err := shared.GetProjectFromSubdomain(subdomain)
334	334		if err != nil {
335	335		logger.Info(
336	336		"could not determine project from subdomain",

...	...		@@ -13,6 +13,23 @@ import (
13	13		"github.com/picosh/utils"
14	14		)
15	15
	16	+	type SubdomainProps struct {
	17	+	ProjectName string
	18	+	Username string
	19	+	}
	20	+
	21	+	func GetProjectFromSubdomain(subdomain string) (*SubdomainProps, error) {
	22	+	props := &SubdomainProps{}
	23	+	strs := strings.SplitN(subdomain, "-", 2)
	24	+	props.Username = strs[0]
	25	+	if len(strs) == 2 {
	26	+	props.ProjectName = strs[1]
	27	+	} else {
	28	+	props.ProjectName = props.Username
	29	+	}
	30	+	return props, nil
	31	+	}
	32	+
16	33		func CorsHeaders(headers http.Header) {
17	34		headers.Add("Access-Control-Allow-Origin", "*")
18	35		headers.Add("Vary", "Origin")

pico

Patchset 35.1 on 2024-11-15T15:03:31Z · commit 77aaa29

Patchset `35.1` on 2024-11-15T15:03:31Z · commit 77aaa29