Logs
Patchsets
Patchset ps-74
reactor(metric-drain): use caddy json format
Eric Bower
auth/api.go
+92
-4
caddy.json
+40
-0
pgs/tunnel.go
+1
-1
pgs/web.go
+2
-19
shared/api.go
+17
-0
reactor(metric-drain): use caddy json format
auth/api.go
link
+92
-4
+92
-4
1diff --git a/auth/api.go b/auth/api.go
2index aad4962..c6fef59 100644
3--- a/auth/api.go
4+++ b/auth/api.go
5@@ -642,6 +642,89 @@ func handler(routes []shared.Route, client *Client) http.HandlerFunc {
6 }
7 }
8
9+type AccessLogReq struct {
10+ RemoteIP string `json:"remote_ip"`
11+ RemotePort string `json:"remote_port"`
12+ ClientIP string `json:"client_ip"`
13+ Method string `json:"method"`
14+ Host string `json:"host"`
15+ Uri string `json:"uri"`
16+ Headers struct {
17+ UserAgent string `json:"User-Agent"`
18+ Referer string `json:"Referer"`
19+ } `json:"headers"`
20+ Tls struct {
21+ ServerName string `json:"server_name"`
22+ } `json:"tls"`
23+}
24+
25+type CaddyAccessLog struct {
26+ Request AccessLogReq `json:"request"`
27+ Status int `json:"status"`
28+}
29+
30+func deserializeCaddyAccessLog(dbpool db.DB, access *CaddyAccessLog) (*db.AnalyticsVisits, error) {
31+ spaceRaw := strings.SplitN(access.Request.Tls.ServerName, ".", 2)
32+ space := spaceRaw[0]
33+ host := access.Request.Host
34+ path := access.Request.Uri
35+ subdomain := ""
36+
37+ // grab subdomain based on host
38+ if strings.HasSuffix(host, "tuns.sh") {
39+ subdomain = strings.TrimSuffix(host, ".tuns.sh")
40+ } else if strings.HasSuffix(host, "pgs.sh") {
41+ subdomain = strings.TrimSuffix(host, ".pgs.sh")
42+ } else if strings.HasSuffix(host, "prose.sh") {
43+ subdomain = strings.TrimSuffix(host, ".prose.sh")
44+ } else {
45+ subdomain = shared.GetCustomDomain(host, space)
46+ }
47+
48+ // get user and namespace details from subdomain
49+ props, err := shared.GetProjectFromSubdomain(subdomain)
50+ if err != nil {
51+ return nil, err
52+ }
53+ // get user ID
54+ user, err := dbpool.FindUserForName(props.Username)
55+ if err != nil {
56+ return nil, err
57+ }
58+
59+ projectID := ""
60+ postID := ""
61+ if space == "pgs" { // figure out project ID
62+ project, err := dbpool.FindProjectByName(user.ID, props.ProjectName)
63+ if err != nil {
64+ return nil, err
65+ }
66+ projectID = project.ID
67+ } else if space == "prose" { // figure out post ID
68+ if path == "" || path == "/" {
69+ } else {
70+ post, err := dbpool.FindPostWithSlug(path, user.ID, space)
71+ if err != nil {
72+ return nil, err
73+ }
74+ postID = post.ID
75+ }
76+ }
77+
78+ return &db.AnalyticsVisits{
79+ UserID: user.ID,
80+ ProjectID: projectID,
81+ PostID: postID,
82+ Namespace: space,
83+ Host: host,
84+ Path: path,
85+ IpAddress: access.Request.ClientIP,
86+ UserAgent: access.Request.Headers.UserAgent,
87+ Referer: access.Request.Headers.Referer, // TODO: I don't see referer in the access log
88+ Status: access.Status,
89+ }, nil
90+}
91+
92 func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secret string) {
93 conn := shared.NewPicoPipeClient()
94 stdoutPipe, err := pubsub.RemoteSub("sub metric-drain -k", ctx, conn)
95@@ -654,14 +737,19 @@ func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secr
96 scanner := bufio.NewScanner(stdoutPipe)
97 for scanner.Scan() {
98 line := scanner.Text()
99- visit := db.AnalyticsVisits{}
100- err := json.Unmarshal([]byte(line), &visit)
101+ accessLog := CaddyAccessLog{}
102+ err := json.Unmarshal([]byte(line), &accessLog)
103 if err != nil {
104 logger.Error("json unmarshal", "err", err)
105 continue
106 }
107
108- err = shared.AnalyticsVisitFromVisit(&visit, dbpool, secret)
109+ visit, err := deserializeCaddyAccessLog(dbpool, &accessLog)
110+ if err != nil {
111+ logger.Error("cannot deserialize access log", "err", err)
112+ continue
113+ }
114+ err = shared.AnalyticsVisitFromVisit(visit, dbpool, secret)
115 if err != nil {
116 if !errors.Is(err, shared.ErrAnalyticsDisabled) {
117 logger.Info("could not record analytics visit", "reason", err)
118@@ -669,7 +757,7 @@ func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secr
119 }
120
121 logger.Info("inserting visit", "visit", visit)
122- err = dbpool.InsertVisit(&visit)
123+ err = dbpool.InsertVisit(visit)
124 if err != nil {
125 logger.Error("could not insert visit record", "err", err)
126 }
caddy.json
link
+40
-0
+40
-0
1diff --git a/caddy.json b/caddy.json
2new file mode 100644
3index 0000000..49e80ec
4--- /dev/null
5+++ b/caddy.json
6@@ -0,0 +1,40 @@
7+{
8+ "level": "info",
9+ "ts": 1731644477.313701,
10+ "logger": "http.log.access",
11+ "msg": "handled request",
12+ "request": {
13+ "remote_ip": "127.0.0.1",
14+ "remote_port": "40400",
15+ "client_ip": "127.0.0.1",
16+ "proto": "HTTP/2.0",
17+ "method": "GET",
18+ "host": "pgs.sh",
19+ "uri": "/",
20+ "headers": { "User-Agent": ["Blackbox Exporter/0.24.0"] },
21+ "tls": {
22+ "resumed": false,
23+ "version": 772,
24+ "cipher_suite": 4865,
25+ "proto": "h2",
26+ "server_name": "pgs.sh"
27+ }
28+ },
29+ "bytes_read": 0,
30+ "user_id": "",
31+ "duration": 0.001207084,
32+ "size": 3718,
33+ "status": 200,
34+ "resp_headers": {
35+ "Referrer-Policy": ["no-referrer-when-downgrade"],
36+ "Strict-Transport-Security": ["max-age=31536000;"],
37+ "X-Content-Type-Options": ["nosniff"],
38+ "X-Frame-Options": ["DENY"],
39+ "Server": ["Caddy"],
40+ "Alt-Svc": ["h3=\":443\"; ma=2592000"],
41+ "Date": ["Fri, 15 Nov 2024 04:21:17 GMT"],
42+ "Content-Type": ["text/html; charset=utf-8"],
43+ "X-Xss-Protection": ["1; mode=block"],
44+ "Permissions-Policy": ["interest-cohort=()"]
45+ }
46+}
pgs/tunnel.go
link
+1
-1
+1
-1
1diff --git a/pgs/tunnel.go b/pgs/tunnel.go
2index b635c8e..accacc5 100644
3--- a/pgs/tunnel.go
4+++ b/pgs/tunnel.go
5@@ -51,7 +51,7 @@ func createHttpHandler(apiConfig *shared.ApiConfig) CtxHttpBridge {
6 "pubkey", pubkeyStr,
7 )
8
9- props, err := getProjectFromSubdomain(subdomain)
10+ props, err := shared.GetProjectFromSubdomain(subdomain)
11 if err != nil {
12 log.Error(err.Error())
13 return http.HandlerFunc(shared.UnauthorizedHandler)
pgs/web.go
link
+2
-19
+2
-19
1diff --git a/pgs/web.go b/pgs/web.go
2index cd251f0..6f3aceb 100644
3--- a/pgs/web.go
4+++ b/pgs/web.go
5@@ -174,7 +174,7 @@ func (web *WebRouter) checkHandler(w http.ResponseWriter, r *http.Request) {
6
7 if !strings.Contains(hostDomain, appDomain) {
8 subdomain := shared.GetCustomDomain(hostDomain, cfg.Space)
9- props, err := getProjectFromSubdomain(subdomain)
10+ props, err := shared.GetProjectFromSubdomain(subdomain)
11 if err != nil {
12 logger.Error(
13 "could not get project from subdomain",
14@@ -330,7 +330,7 @@ func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, fro
15 "host", r.Host,
16 )
17
18- props, err := getProjectFromSubdomain(subdomain)
19+ props, err := shared.GetProjectFromSubdomain(subdomain)
20 if err != nil {
21 logger.Info(
22 "could not determine project from subdomain",
23@@ -447,20 +447,3 @@ func (web *WebRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
24 ctx = context.WithValue(ctx, shared.CtxSubdomainKey{}, subdomain)
25 router.ServeHTTP(w, r.WithContext(ctx))
26 }
27-
28-type SubdomainProps struct {
29- ProjectName string
30- Username string
31-}
32-
33-func getProjectFromSubdomain(subdomain string) (*SubdomainProps, error) {
34- props := &SubdomainProps{}
35- strs := strings.SplitN(subdomain, "-", 2)
36- props.Username = strs[0]
37- if len(strs) == 2 {
38- props.ProjectName = strs[1]
39- } else {
40- props.ProjectName = props.Username
41- }
42- return props, nil
43-}