pico

reactor(metric-drain): use caddy json format open rss
created pr with 35.1 on 2024-11-15T15:03:31Z · by c8ef7d19
added 35.2 on 2024-11-23T03:34:44Z · by c8ef7d19
1: 77aaa29 ! 1: 8d56535 reactor(metric-drain): use caddy json format
-: ------- > 2: a336041 wip
-: ------- > 3: 7ae45b3 chore: wrap
-: ------- > 4: bfa5c4f done
added 35.3 on 2024-11-27T20:17:20Z · by c8ef7d19
1: 8d56535 < -: ------- reactor(metric-drain): use caddy json format
-: ------- > 1: c7eeb12 reactor(metric-drain): use caddy access logs
2: a336041 < -: ------- wip
3: 7ae45b3 < -: ------- chore: wrap
4: bfa5c4f < -: ------- done
added 35.4 on 2024-11-27T20:18:31Z · by c8ef7d19
1: c7eeb12 ! 1: 4e0839a reactor(metric-drain): use caddy access logs
changed status to accepted on 2024-11-28T03:03:53Z · by c8ef7d19
cmds
checkout latest patchset: 
ssh pr.pico.sh print 35 | git am -3
checkout any patchset in a patch request: 
ssh pr.pico.sh print 35.[rev] | git am -3
add changes to patch request: 
git format-patch main --stdout | ssh pr.pico.sh pr add 35
set PR to open (enables RSS notifications): 
ssh pr.pico.sh pr open 35
set PR to draft (stops RSS notifications): 
ssh pr.pico.sh pr draft 35

Patchset 35.2 on 2024-11-23T03:34:44Z · commit 8d56535

→ reactor(metric-drain): use caddy json format
Eric Bower 2024-11-15T15:02:24Z
 wip
Eric Bower 2024-11-23T02:06:47Z
 chore: wrap
Eric Bower 2024-11-23T02:41:15Z
 done
Eric Bower 2024-11-23T03:32:31Z
+106 -26 auth/api.go #
......@@ -14,6 +14,7 @@ import (
1414 "log/slog"
1515 "net/http"
1616 "net/url"
17+ "strings"
1718 "time"
1819
1920 "github.com/gorilla/feeds"
......@@ -578,6 +579,89 @@ func checkoutHandler() http.HandlerFunc {
578579 }
579580 }
580581
582+type AccessLogReq struct {
583+ RemoteIP string `json:"remote_ip"`
584+ RemotePort string `json:"remote_port"`
585+ ClientIP string `json:"client_ip"`
586+ Method string `json:"method"`
587+ Host string `json:"host"`
588+ Uri string `json:"uri"`
589+ Headers struct {
590+ UserAgent string `json:"User-Agent"`
591+ Referer string `json:"Referer"`
592+ } `json:"headers"`
593+ Tls struct {
594+ ServerName string `json:"server_name"`
595+ } `json:"tls"`
596+}
597+
598+type CaddyAccessLog struct {
599+ Request AccessLogReq `json:"request"`
600+ Status int `json:"status"`
601+}
602+
603+func deserializeCaddyAccessLog(dbpool db.DB, access *CaddyAccessLog) (*db.AnalyticsVisits, error) {
604+ spaceRaw := strings.SplitN(access.Request.Tls.ServerName, ".", 2)
605+ space := spaceRaw[0]
606+ host := access.Request.Host
607+ path := access.Request.Uri
608+ subdomain := ""
609+
610+ // grab subdomain based on host
611+ if strings.HasSuffix(host, "tuns.sh") {
612+ subdomain = strings.TrimSuffix(host, ".tuns.sh")
613+ } else if strings.HasSuffix(host, "pgs.sh") {
614+ subdomain = strings.TrimSuffix(host, ".pgs.sh")
615+ } else if strings.HasSuffix(host, "prose.sh") {
616+ subdomain = strings.TrimSuffix(host, ".prose.sh")
617+ } else {
618+ subdomain = shared.GetCustomDomain(host, space)
619+ }
620+
621+ // get user and namespace details from subdomain
622+ props, err := shared.GetProjectFromSubdomain(subdomain)
623+ if err != nil {
624+ return nil, err
625+ }
626+ // get user ID
627+ user, err := dbpool.FindUserForName(props.Username)
628+ if err != nil {
629+ return nil, err
630+ }
631+
632+ projectID := ""
633+ postID := ""
634+ if space == "pgs" { // figure out project ID
635+ project, err := dbpool.FindProjectByName(user.ID, props.ProjectName)
636+ if err != nil {
637+ return nil, err
638+ }
639+ projectID = project.ID
640+ } else if space == "prose" { // figure out post ID
641+ if path == "" || path == "/" {
642+ } else {
643+ post, err := dbpool.FindPostWithSlug(path, user.ID, space)
644+ if err != nil {
645+ return nil, err
646+ }
647+ postID = post.ID
648+ }
649+ }
650+
651+ return &db.AnalyticsVisits{
652+ UserID: user.ID,
653+ ProjectID: projectID,
654+ PostID: postID,
655+ Namespace: space,
656+ Host: host,
657+ Path: path,
658+ IpAddress: access.Request.ClientIP,
659+ UserAgent: access.Request.Headers.UserAgent,
660+ Referer: access.Request.Headers.Referer, // TODO: I don't see referer in the access log
661+ Status: access.Status,
662+ }, nil
663+}
664+
581665 func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secret string) {
582666 drain := metrics.ReconnectReadMetrics(
583667 ctx,
......@@ -587,36 +671,32 @@ func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secr
587671 -1,
588672 )
589673
590- for {
591- scanner := bufio.NewScanner(drain)
592- for scanner.Scan() {
593- line := scanner.Text()
594- visit := db.AnalyticsVisits{}
595- err := json.Unmarshal([]byte(line), &visit)
596- if err != nil {
597- logger.Error("json unmarshal", "err", err)
598- continue
599- }
600-
601- user := slog.Any("userId", visit.UserID)
602-
603- err = shared.AnalyticsVisitFromVisit(&visit, dbpool, secret)
604- if err != nil {
605- if !errors.Is(err, shared.ErrAnalyticsDisabled) {
606- logger.Info("could not record analytics visit", "reason", err, "visit", visit, user)
607- continue
608- }
609- }
674+ scanner := bufio.NewScanner(drain)
675+ for scanner.Scan() {
676+ line := scanner.Text()
677+ accessLog := CaddyAccessLog{}
678+ err := json.Unmarshal([]byte(line), &accessLog)
679+ if err != nil {
680+ logger.Error("json unmarshal", "err", err)
681+ continue
682+ }
610683
611- logger.Info("inserting visit", "visit", visit, user)
612- err = dbpool.InsertVisit(&visit)
613- if err != nil {
614- logger.Error("could not insert visit record", "err", err, "visit", visit, user)
684+ visit, err := deserializeCaddyAccessLog(dbpool, &accessLog)
685+ if err != nil {
686+ logger.Error("cannot deserialize access log", "err", err)
687+ continue
688+ }
689+ err = shared.AnalyticsVisitFromVisit(visit, dbpool, secret)
690+ if err != nil {
691+ if !errors.Is(err, shared.ErrAnalyticsDisabled) {
692+ logger.Info("could not record analytics visit", "reason", err)
615693 }
616694 }
617695
618- if scanner.Err() != nil {
619- logger.Error("scanner error", "err", scanner.Err())
696+ logger.Info("inserting visit", "visit", visit)
697+ err = dbpool.InsertVisit(visit)
698+ if err != nil {
699+ logger.Error("could not insert visit record", "err", err)
620700 }
621701 }
622702 }
+40 -0 caddy.json #
......@@ -0,0 +1,40 @@
1+{
2+ "level": "info",
3+ "ts": 1731644477.313701,
4+ "logger": "http.log.access",
5+ "msg": "handled request",
6+ "request": {
7+ "remote_ip": "127.0.0.1",
8+ "remote_port": "40400",
9+ "client_ip": "127.0.0.1",
10+ "proto": "HTTP/2.0",
11+ "method": "GET",
12+ "host": "pgs.sh",
13+ "uri": "/",
14+ "headers": { "User-Agent": ["Blackbox Exporter/0.24.0"] },
15+ "tls": {
16+ "resumed": false,
17+ "version": 772,
18+ "cipher_suite": 4865,
19+ "proto": "h2",
20+ "server_name": "pgs.sh"
21+ }
22+ },
23+ "bytes_read": 0,
24+ "user_id": "",
25+ "duration": 0.001207084,
26+ "size": 3718,
27+ "status": 200,
28+ "resp_headers": {
29+ "Referrer-Policy": ["no-referrer-when-downgrade"],
30+ "Strict-Transport-Security": ["max-age=31536000;"],
31+ "X-Content-Type-Options": ["nosniff"],
32+ "X-Frame-Options": ["DENY"],
33+ "Server": ["Caddy"],
34+ "Alt-Svc": ["h3=\":443\"; ma=2592000"],
35+ "Date": ["Fri, 15 Nov 2024 04:21:17 GMT"],
36+ "Content-Type": ["text/html; charset=utf-8"],
37+ "X-Xss-Protection": ["1; mode=block"],
38+ "Permissions-Policy": ["interest-cohort=()"]
39+ }
40+}
+1 -1 pgs/tunnel.go #
......@@ -51,7 +51,7 @@ func createHttpHandler(apiConfig *shared.ApiConfig) CtxHttpBridge {
5151 "pubkey", pubkeyStr,
5252 )
5353
54- props, err := getProjectFromSubdomain(subdomain)
54+ props, err := shared.GetProjectFromSubdomain(subdomain)
5555 if err != nil {
5656 log.Error(err.Error())
5757 return http.HandlerFunc(shared.UnauthorizedHandler)
+2 -19 pgs/web.go #
......@@ -177,7 +177,7 @@ func (web *WebRouter) checkHandler(w http.ResponseWriter, r *http.Request) {
177177
178178 if !strings.Contains(hostDomain, appDomain) {
179179 subdomain := shared.GetCustomDomain(hostDomain, cfg.Space)
180- props, err := getProjectFromSubdomain(subdomain)
180+ props, err := shared.GetProjectFromSubdomain(subdomain)
181181 if err != nil {
182182 logger.Error(
183183 "could not get project from subdomain",
......@@ -333,7 +333,7 @@ func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, fro
333333 "host", r.Host,
334334 )
335335
336- props, err := getProjectFromSubdomain(subdomain)
336+ props, err := shared.GetProjectFromSubdomain(subdomain)
337337 if err != nil {
338338 logger.Info(
339339 "could not determine project from subdomain",
......@@ -450,20 +450,3 @@ func (web *WebRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
450450 ctx = context.WithValue(ctx, shared.CtxSubdomainKey{}, subdomain)
451451 router.ServeHTTP(w, r.WithContext(ctx))
452452 }
453-
454-type SubdomainProps struct {
455- ProjectName string
456- Username string
457-}
458-
459-func getProjectFromSubdomain(subdomain string) (*SubdomainProps, error) {
460- props := &SubdomainProps{}
461- strs := strings.SplitN(subdomain, "-", 2)
462- props.Username = strs[0]
463- if len(strs) == 2 {
464- props.ProjectName = strs[1]
465- } else {
466- props.ProjectName = props.Username
467- }
468- return props, nil
469-}
+17 -0 shared/api.go #
......@@ -13,6 +13,23 @@ import (
1313 "github.com/picosh/utils"
1414 )
1515
16+type SubdomainProps struct {
17+ ProjectName string
18+ Username string
19+}
20+
21+func GetProjectFromSubdomain(subdomain string) (*SubdomainProps, error) {
22+ props := &SubdomainProps{}
23+ strs := strings.SplitN(subdomain, "-", 2)
24+ props.Username = strs[0]
25+ if len(strs) == 2 {
26+ props.ProjectName = strs[1]
27+ } else {
28+ props.ProjectName = props.Username
29+ }
30+ return props, nil
31+}
32+
1633 func CorsHeaders(headers http.Header) {
1734 headers.Add("Access-Control-Allow-Origin", "*")
1835 headers.Add("Vary", "Origin")
Back to top