pico

reactor(metric-drain): use caddy json format open rss
created pr with 35.1 on 2024-11-15T15:03:31Z · by c8ef7d19
added 35.2 on 2024-11-23T03:34:44Z · by c8ef7d19
1: 77aaa29 ! 1: 8d56535 reactor(metric-drain): use caddy json format
-: ------- > 2: a336041 wip
-: ------- > 3: 7ae45b3 chore: wrap
-: ------- > 4: bfa5c4f done
added 35.3 on 2024-11-27T20:17:20Z · by c8ef7d19
1: 8d56535 < -: ------- reactor(metric-drain): use caddy json format
-: ------- > 1: c7eeb12 reactor(metric-drain): use caddy access logs
2: a336041 < -: ------- wip
3: 7ae45b3 < -: ------- chore: wrap
4: bfa5c4f < -: ------- done
added 35.4 on 2024-11-27T20:18:31Z · by c8ef7d19
1: c7eeb12 ! 1: 4e0839a reactor(metric-drain): use caddy access logs
changed status to accepted on 2024-11-28T03:03:53Z · by c8ef7d19
cmds
checkout latest patchset: 
ssh pr.pico.sh print 35 | git am -3
checkout any patchset in a patch request: 
ssh pr.pico.sh print 35.[rev] | git am -3
add changes to patch request: 
git format-patch main --stdout | ssh pr.pico.sh pr add 35
set PR to open (enables RSS notifications): 
ssh pr.pico.sh pr open 35
set PR to draft (stops RSS notifications): 
ssh pr.pico.sh pr draft 35

Patchset 35.3 on 2024-11-27T20:17:20Z · commit c7eeb12

→ reactor(metric-drain): use caddy access logs
Eric Bower 2024-11-15T15:02:24Z
reactor(metric-drain): use caddy access logs
Eric Bower 2024-11-15T15:02:24Z
Previously we were sending site usage analytics within our web app code.
This worked well for our use case because we could filter, parse, and
send the analytics to our pipe `metric-drain` which would then store the
analytics into our database.

Because we want to enable HTTP caching for pgs we won't always reach our
web app code since usage analytics will terminate at our cache layer.

Instead, we want to record analytics higher in the request stack.  In
this case, we want to record site analytics from Caddy access logs.

Here's how it works:

- `pub` caddy access logs to our pipe `container-drain`
- `auth/web` will `sub` to `container-drain`, filter, deserialize, and
  `pub` to `metric-drain`
- `auth/web` will `sub` to `metric-drain` and store the analytics in our
  database
Semantic diff summary
8 added, 27 modified, 1 signature changed, 7 removed across 13 analyzed files (4 files skipped: unsupported file type)
auth/api.go
db/db.go
db/postgres/storage.go
imgs/api.go
pastes/api.go
pgs/ssh.go
pgs/tunnel.go
pgs/web.go
pgs/web_asset_handler.go
pgs/web_test.go
prose/api.go
shared/api.go
shared/router.go
+2 -1 Makefile #
......@@ -135,10 +135,11 @@ migrate:
135135 $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20240819_add_projects_blocked.sql
136136 $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241028_add_analytics_indexes.sql
137137 $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241114_add_namespace_to_analytics.sql
138+ $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241125_add_content_type_to_analytics.sql
138139 .PHONY: migrate
139140
140141 latest:
141- $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241114_add_namespace_to_analytics.sql
142+ $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20241125_add_content_type_to_analytics.sql
142143 .PHONY: latest
143144
144145 psql:
+162 -12 auth/api.go #
......@@ -14,6 +14,7 @@ import (
1414 "log/slog"
1515 "net/http"
1616 "net/url"
17+ "strings"
1718 "time"
1819
1920 "github.com/gorilla/feeds"
......@@ -21,6 +22,7 @@ import (
2122 "github.com/picosh/pico/db/postgres"
2223 "github.com/picosh/pico/shared"
2324 "github.com/picosh/utils"
25+ "github.com/picosh/utils/pipe"
2426 "github.com/picosh/utils/pipe/metrics"
2527 )
2628
......@@ -578,6 +580,155 @@ func checkoutHandler() http.HandlerFunc {
578580 }
579581 }
580582
583+type AccessLogReq struct {
584+ RemoteIP string `json:"remote_ip"`
585+ RemotePort string `json:"remote_port"`
586+ ClientIP string `json:"client_ip"`
587+ Method string `json:"method"`
588+ Host string `json:"host"`
589+ Uri string `json:"uri"`
590+ Headers struct {
591+ UserAgent []string `json:"User-Agent"`
592+ Referer []string `json:"Referer"`
593+ } `json:"headers"`
594+ Tls struct {
595+ ServerName string `json:"server_name"`
596+ } `json:"tls"`
597+}
598+
599+type RespHeaders struct {
600+ ContentType []string `json:"Content-Type"`
601+}
602+
603+type CaddyAccessLog struct {
604+ Request AccessLogReq `json:"request"`
605+ Status int `json:"status"`
606+ RespHeaders RespHeaders `json:"resp_headers"`
607+}
608+
609+func deserializeCaddyAccessLog(dbpool db.DB, access *CaddyAccessLog) (*db.AnalyticsVisits, error) {
610+ spaceRaw := strings.SplitN(access.Request.Tls.ServerName, ".", 2)
611+ space := spaceRaw[0]
612+ host := access.Request.Host
613+ path := access.Request.Uri
614+ subdomain := ""
615+
616+ // grab subdomain based on host
617+ if strings.HasSuffix(host, "tuns.sh") {
618+ subdomain = strings.TrimSuffix(host, ".tuns.sh")
619+ } else if strings.HasSuffix(host, "pgs.sh") {
620+ subdomain = strings.TrimSuffix(host, ".pgs.sh")
621+ } else if strings.HasSuffix(host, "prose.sh") {
622+ subdomain = strings.TrimSuffix(host, ".prose.sh")
623+ } else {
624+ subdomain = shared.GetCustomDomain(host, space)
625+ }
626+
627+ // get user and namespace details from subdomain
628+ props, err := shared.GetProjectFromSubdomain(subdomain)
629+ if err != nil {
630+ return nil, err
631+ }
632+ // get user ID
633+ user, err := dbpool.FindUserForName(props.Username)
634+ if err != nil {
635+ return nil, err
636+ }
637+
638+ projectID := ""
639+ postID := ""
640+ if space == "pgs" { // figure out project ID
641+ project, err := dbpool.FindProjectByName(user.ID, props.ProjectName)
642+ if err != nil {
643+ return nil, err
644+ }
645+ projectID = project.ID
646+ } else if space == "prose" { // figure out post ID
647+ if path == "" || path == "/" {
648+ } else {
649+ post, err := dbpool.FindPostWithSlug(path, user.ID, space)
650+ if err != nil {
651+ return nil, err
652+ }
653+ postID = post.ID
654+ }
655+ }
656+
657+ return &db.AnalyticsVisits{
658+ UserID: user.ID,
659+ ProjectID: projectID,
660+ PostID: postID,
661+ Namespace: space,
662+ Host: host,
663+ Path: path,
664+ IpAddress: access.Request.ClientIP,
665+ UserAgent: strings.Join(access.Request.Headers.UserAgent, " "),
666+ Referer: strings.Join(access.Request.Headers.Referer, " "),
667+ ContentType: strings.Join(access.RespHeaders.ContentType, " "),
668+ Status: access.Status,
669+ }, nil
670+}
671+
672+// this feels really stupid because i'm taking containter-drain,
673+// filtering it, and then sending it to metric-drain. The
674+// metricDrainSub function listens on the metric-drain and saves it.
675+// So why not just call the necessary functions to save the visit?
676+// We want to be able to use pipe as a debugging tool which means we
677+// can manually sub to `metric-drain` and have a nice clean output to view.
678+func containerDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger) {
679+ info := shared.NewPicoPipeClient()
680+ drain := pipe.NewReconnectReadWriteCloser(
681+ ctx,
682+ logger,
683+ info,
684+ "container drain",
685+ "sub container-drain -k",
686+ 100,
687+ -1,
688+ )
689+
690+ send := pipe.NewReconnectReadWriteCloser(
691+ ctx,
692+ logger,
693+ info,
694+ "from container drain to metric drain",
695+ "pub metric-drain -b=false",
696+ 100,
697+ -1,
698+ )
699+
700+ for {
701+ scanner := bufio.NewScanner(drain)
702+ for scanner.Scan() {
703+ line := scanner.Text()
704+ if strings.Contains(line, "http.log.access") {
705+ clean := strings.TrimSpace(line)
706+ visit, err := accessLogToVisit(dbpool, clean)
707+ if err != nil {
708+ logger.Debug("could not convert access log to a visit", "err", err)
709+ continue
710+ }
711+ jso, err := json.Marshal(visit)
712+ if err != nil {
713+ logger.Error("could not marshal json of a visit", "err", err)
714+ continue
715+ }
716+ _, _ = send.Write(jso)
717+ }
718+ }
719+ }
720+}
721+
722+func accessLogToVisit(dbpool db.DB, line string) (*db.AnalyticsVisits, error) {
723+ accessLog := CaddyAccessLog{}
724+ err := json.Unmarshal([]byte(line), &accessLog)
725+ if err != nil {
726+ return nil, err
727+ }
728+
729+ return deserializeCaddyAccessLog(dbpool, &accessLog)
730+}
731+
581732 func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secret string) {
582733 drain := metrics.ReconnectReadMetrics(
583734 ctx,
......@@ -594,30 +745,26 @@ func metricDrainSub(ctx context.Context, dbpool db.DB, logger *slog.Logger, secr
594745 visit := db.AnalyticsVisits{}
595746 err := json.Unmarshal([]byte(line), &visit)
596747 if err != nil {
597- logger.Error("json unmarshal", "err", err)
748+ logger.Info("could not unmarshal json", "err", err, "line", line)
598749 continue
599750 }
600-
601- user := slog.Any("userId", visit.UserID)
602-
603751 err = shared.AnalyticsVisitFromVisit(&visit, dbpool, secret)
604752 if err != nil {
605753 if !errors.Is(err, shared.ErrAnalyticsDisabled) {
606- logger.Info("could not record analytics visit", "reason", err, "visit", visit, user)
607- continue
754+ logger.Info("could not record analytics visit", "reason", err)
608755 }
609756 }
610757
611- logger.Info("inserting visit", "visit", visit, user)
758+ if visit.ContentType != "" && !strings.HasPrefix(visit.ContentType, "text/html") {
759+ continue
760+ }
761+
762+ logger.Info("inserting visit", "visit", visit)
612763 err = dbpool.InsertVisit(&visit)
613764 if err != nil {
614- logger.Error("could not insert visit record", "err", err, "visit", visit, user)
765+ logger.Error("could not insert visit record", "err", err)
615766 }
616767 }
617-
618- if scanner.Err() != nil {
619- logger.Error("scanner error", "err", scanner.Err())
620- }
621768 }
622769 }
623770
......@@ -689,6 +836,9 @@ func StartApiServer() {
689836
690837 // gather metrics in the auth service
691838 go metricDrainSub(ctx, db, logger, cfg.Secret)
839+ // convert container logs to access logs
840+ go containerDrainSub(ctx, db, logger)
841+
692842 defer ctx.Done()
693843
694844 apiConfig := &shared.ApiConfig{
+40 -0 caddy.json #
......@@ -0,0 +1,40 @@
1+{
2+ "level": "info",
3+ "ts": 1731644477.313701,
4+ "logger": "http.log.access",
5+ "msg": "handled request",
6+ "request": {
7+ "remote_ip": "127.0.0.1",
8+ "remote_port": "40400",
9+ "client_ip": "127.0.0.1",
10+ "proto": "HTTP/2.0",
11+ "method": "GET",
12+ "host": "pgs.sh",
13+ "uri": "/",
14+ "headers": { "User-Agent": ["Blackbox Exporter/0.24.0"] },
15+ "tls": {
16+ "resumed": false,
17+ "version": 772,
18+ "cipher_suite": 4865,
19+ "proto": "h2",
20+ "server_name": "pgs.sh"
21+ }
22+ },
23+ "bytes_read": 0,
24+ "user_id": "",
25+ "duration": 0.001207084,
26+ "size": 3718,
27+ "status": 200,
28+ "resp_headers": {
29+ "Referrer-Policy": ["no-referrer-when-downgrade"],
30+ "Strict-Transport-Security": ["max-age=31536000;"],
31+ "X-Content-Type-Options": ["nosniff"],
32+ "X-Frame-Options": ["DENY"],
33+ "Server": ["Caddy"],
34+ "Alt-Svc": ["h3=\":443\"; ma=2592000"],
35+ "Date": ["Fri, 15 Nov 2024 04:21:17 GMT"],
36+ "Content-Type": ["text/html; charset=utf-8"],
37+ "X-Xss-Protection": ["1; mode=block"],
38+ "Permissions-Policy": ["interest-cohort=()"]
39+ }
40+}
+12 -11 db/db.go #
......@@ -161,17 +161,18 @@ type PostAnalytics struct {
161161 }
162162
163163 type AnalyticsVisits struct {
164- ID string `json:"id"`
165- UserID string `json:"user_id"`
166- ProjectID string `json:"project_id"`
167- PostID string `json:"post_id"`
168- Namespace string `json:"namespace"`
169- Host string `json:"host"`
170- Path string `json:"path"`
171- IpAddress string `json:"ip_address"`
172- UserAgent string `json:"user_agent"`
173- Referer string `json:"referer"`
174- Status int `json:"status"`
164+ ID string `json:"id"`
165+ UserID string `json:"user_id"`
166+ ProjectID string `json:"project_id"`
167+ PostID string `json:"post_id"`
168+ Namespace string `json:"namespace"`
169+ Host string `json:"host"`
170+ Path string `json:"path"`
171+ IpAddress string `json:"ip_address"`
172+ UserAgent string `json:"user_agent"`
173+ Referer string `json:"referer"`
174+ Status int `json:"status"`
175+ ContentType string `json:"content_type"`
175176 }
176177
177178 type VisitInterval struct {
+2 -1 db/postgres/storage.go #
......@@ -986,7 +986,7 @@ func newNullString(s string) sql.NullString {
986986
987987 func (me *PsqlDB) InsertVisit(visit *db.AnalyticsVisits) error {
988988 _, err := me.Db.Exec(
989- `INSERT INTO analytics_visits (user_id, project_id, post_id, namespace, host, path, ip_address, user_agent, referer, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);`,
989+ `INSERT INTO analytics_visits (user_id, project_id, post_id, namespace, host, path, ip_address, user_agent, referer, status, content_type) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11);`,
990990 visit.UserID,
991991 newNullString(visit.ProjectID),
992992 newNullString(visit.PostID),
......@@ -997,6 +997,7 @@ func (me *PsqlDB) InsertVisit(visit *db.AnalyticsVisits) error {
997997 visit.UserAgent,
998998 visit.Referer,
999999 visit.Status,
1000+ visit.ContentType,
10001001 )
10011002 return err
10021003 }
+0 -2 imgs/api.go #
......@@ -177,7 +177,6 @@ func ImgRequest(w http.ResponseWriter, r *http.Request) {
177177 dbpool := shared.GetDB(r)
178178 logger := shared.GetLogger(r)
179179 username := shared.GetUsernameFromRequest(r)
180- analytics := shared.GetAnalyticsQueue(r)
181180
182181 user, err := dbpool.FindUserForName(username)
183182 if err != nil {
......@@ -241,7 +240,6 @@ func ImgRequest(w http.ResponseWriter, r *http.Request) {
241240 logger,
242241 dbpool,
243242 st,
244- analytics,
245243 )
246244 router.ServeAsset(fname, opts, true, anyPerm, w, r)
247245 }
+0 -5 pastes/api.go #
......@@ -59,11 +59,6 @@ type PostPageData struct {
5959 Unlisted bool
6060 }
6161
62-type TransparencyPageData struct {
63- Site shared.SitePageData
64- Analytics *db.Analytics
65-}
66-
6762 type Link struct {
6863 URL string
6964 Text string
+3 -7 pgs/ssh.go #
......@@ -11,7 +11,6 @@ import (
1111 "github.com/charmbracelet/promwish"
1212 "github.com/charmbracelet/ssh"
1313 "github.com/charmbracelet/wish"
14- "github.com/picosh/pico/db"
1514 "github.com/picosh/pico/db/postgres"
1615 "github.com/picosh/pico/shared"
1716 "github.com/picosh/pico/shared/storage"
......@@ -81,13 +80,10 @@ func StartSshServer() {
8180 st,
8281 )
8382
84- ch := make(chan *db.AnalyticsVisits, 100)
85- go shared.AnalyticsCollect(ch, dbpool, logger)
8683 apiConfig := &shared.ApiConfig{
87- Cfg: cfg,
88- Dbpool: dbpool,
89- Storage: st,
90- AnalyticsQueue: ch,
84+ Cfg: cfg,
85+ Dbpool: dbpool,
86+ Storage: st,
9187 }
9288
9389 webTunnel := &tunkit.WebTunnelHandler{
+1 -2 pgs/tunnel.go #
......@@ -51,7 +51,7 @@ func createHttpHandler(apiConfig *shared.ApiConfig) CtxHttpBridge {
5151 "pubkey", pubkeyStr,
5252 )
5353
54- props, err := getProjectFromSubdomain(subdomain)
54+ props, err := shared.GetProjectFromSubdomain(subdomain)
5555 if err != nil {
5656 log.Error(err.Error())
5757 return http.HandlerFunc(shared.UnauthorizedHandler)
......@@ -121,7 +121,6 @@ func createHttpHandler(apiConfig *shared.ApiConfig) CtxHttpBridge {
121121 logger,
122122 apiConfig.Dbpool,
123123 apiConfig.Storage,
124- apiConfig.AnalyticsQueue,
125124 )
126125 tunnelRouter := TunnelWebRouter{routes}
127126 router := http.NewServeMux()
+14 -36 pgs/web.go #
......@@ -40,10 +40,7 @@ func StartApiServer() {
4040 return
4141 }
4242
43- ch := make(chan *db.AnalyticsVisits, 100)
44- go shared.AnalyticsCollect(ch, dbpool, logger)
45-
46- routes := NewWebRouter(cfg, logger, dbpool, st, ch)
43+ routes := NewWebRouter(cfg, logger, dbpool, st)
4744
4845 portStr := fmt.Sprintf(":%s", cfg.Port)
4946 logger.Info(
......@@ -61,22 +58,20 @@ func StartApiServer() {
6158 type HasPerm = func(proj *db.Project) bool
6259
6360 type WebRouter struct {
64- Cfg *shared.ConfigSite
65- Logger *slog.Logger
66- Dbpool db.DB
67- Storage storage.StorageServe
68- AnalyticsQueue chan *db.AnalyticsVisits
69- RootRouter *http.ServeMux
70- UserRouter *http.ServeMux
61+ Cfg *shared.ConfigSite
62+ Logger *slog.Logger
63+ Dbpool db.DB
64+ Storage storage.StorageServe
65+ RootRouter *http.ServeMux
66+ UserRouter *http.ServeMux
7167 }
7268
73-func NewWebRouter(cfg *shared.ConfigSite, logger *slog.Logger, dbpool db.DB, st storage.StorageServe, analytics chan *db.AnalyticsVisits) *WebRouter {
69+func NewWebRouter(cfg *shared.ConfigSite, logger *slog.Logger, dbpool db.DB, st storage.StorageServe) *WebRouter {
7470 router := &WebRouter{
75- Cfg: cfg,
76- Logger: logger,
77- Dbpool: dbpool,
78- Storage: st,
79- AnalyticsQueue: analytics,
71+ Cfg: cfg,
72+ Logger: logger,
73+ Dbpool: dbpool,
74+ Storage: st,
8075 }
8176 router.initRouters()
8277 return router
......@@ -177,7 +172,7 @@ func (web *WebRouter) checkHandler(w http.ResponseWriter, r *http.Request) {
177172
178173 if !strings.Contains(hostDomain, appDomain) {
179174 subdomain := shared.GetCustomDomain(hostDomain, cfg.Space)
180- props, err := getProjectFromSubdomain(subdomain)
175+ props, err := shared.GetProjectFromSubdomain(subdomain)
181176 if err != nil {
182177 logger.Error(
183178 "could not get project from subdomain",
......@@ -333,7 +328,7 @@ func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, fro
333328 "host", r.Host,
334329 )
335330
336- props, err := getProjectFromSubdomain(subdomain)
331+ props, err := shared.GetProjectFromSubdomain(subdomain)
337332 if err != nil {
338333 logger.Info(
339334 "could not determine project from subdomain",
......@@ -450,20 +445,3 @@ func (web *WebRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
450445 ctx = context.WithValue(ctx, shared.CtxSubdomainKey{}, subdomain)
451446 router.ServeHTTP(w, r.WithContext(ctx))
452447 }
453-
454-type SubdomainProps struct {
455- ProjectName string
456- Username string
457-}
458-
459-func getProjectFromSubdomain(subdomain string) (*SubdomainProps, error) {
460- props := &SubdomainProps{}
461- strs := strings.SplitN(subdomain, "-", 2)
462- props.Username = strs[0]
463- if len(strs) == 2 {
464- props.ProjectName = strs[1]
465- } else {
466- props.ProjectName = props.Username
467- }
468- return props, nil
469-}
+0 -37 pgs/web_asset_handler.go #
......@@ -1,7 +1,6 @@
11 package pgs
22
33 import (
4- "errors"
54 "fmt"
65 "io"
76 "log/slog"
......@@ -15,7 +14,6 @@ import (
1514 "net/http/httputil"
1615 _ "net/http/pprof"
1716
18- "github.com/picosh/pico/shared"
1917 "github.com/picosh/pico/shared/storage"
2018 sst "github.com/picosh/pobj/storage"
2119 )
......@@ -155,22 +153,6 @@ func (h *ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
155153 "routes", strings.Join(attempts, ", "),
156154 "status", http.StatusNotFound,
157155 )
158- // track 404s
159- ch := h.AnalyticsQueue
160- view, err := shared.AnalyticsVisitFromRequest(r, h.Dbpool, h.UserID)
161- if err == nil {
162- view.ProjectID = h.ProjectID
163- view.Status = http.StatusNotFound
164- select {
165- case ch <- view:
166- default:
167- logger.Error("could not send analytics view to channel", "view", view)
168- }
169- } else {
170- if !errors.Is(err, shared.ErrAnalyticsDisabled) {
171- logger.Error("could not record analytics view", "err", err, "view", view)
172- }
173- }
174156 http.Error(w, "404 not found", http.StatusNotFound)
175157 return
176158 }
......@@ -236,25 +218,6 @@ func (h *ApiAssetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
236218
237219 finContentType := w.Header().Get("content-type")
238220
239- // only track pages, not individual assets
240- if finContentType == "text/html" {
241- // track visit
242- ch := h.AnalyticsQueue
243- view, err := shared.AnalyticsVisitFromRequest(r, h.Dbpool, h.UserID)
244- if err == nil {
245- view.ProjectID = h.ProjectID
246- select {
247- case ch <- view:
248- default:
249- logger.Error("could not send analytics view to channel", "view", view)
250- }
251- } else {
252- if !errors.Is(err, shared.ErrAnalyticsDisabled) {
253- logger.Error("could not record analytics view", "err", err, "view", view)
254- }
255- }
256- }
257-
258221 logger.Info(
259222 "serving asset",
260223 "asset", assetFilepath,
+2 -42 pgs/web_test.go #
......@@ -8,7 +8,6 @@ import (
88 "net/http/httptest"
99 "strings"
1010 "testing"
11- "time"
1211
1312 "github.com/picosh/pico/db"
1413 "github.com/picosh/pico/db/stub"
......@@ -219,8 +218,7 @@ func TestApiBasic(t *testing.T) {
219218 responseRecorder := httptest.NewRecorder()
220219
221220 st, _ := storage.NewStorageMemory(tc.storage)
222- ch := make(chan *db.AnalyticsVisits, 100)
223- router := NewWebRouter(cfg, cfg.Logger, tc.dbpool, st, ch)
221+ router := NewWebRouter(cfg, cfg.Logger, tc.dbpool, st)
224222 router.ServeHTTP(responseRecorder, request)
225223
226224 if responseRecorder.Code != tc.status {
......@@ -240,43 +238,6 @@ func TestApiBasic(t *testing.T) {
240238 }
241239 }
242240
243-func TestAnalytics(t *testing.T) {
244- bucketName := shared.GetAssetBucketName(testUserID)
245- cfg := NewConfigSite()
246- cfg.Domain = "pgs.test"
247- expectedPath := "/app"
248- request := httptest.NewRequest("GET", mkpath(expectedPath), strings.NewReader(""))
249- responseRecorder := httptest.NewRecorder()
250-
251- sto := map[string]map[string]string{
252- bucketName: {
253- "test/app.html": "hello world!",
254- },
255- }
256- st, _ := storage.NewStorageMemory(sto)
257- ch := make(chan *db.AnalyticsVisits, 100)
258- dbpool := NewPgsAnalticsDb(cfg.Logger)
259- router := NewWebRouter(cfg, cfg.Logger, dbpool, st, ch)
260-
261- go func() {
262- for analytics := range ch {
263- if analytics.Path != expectedPath {
264- t.Errorf("Want path '%s', got '%s'", expectedPath, analytics.Path)
265- }
266- close(ch)
267- }
268- }()
269-
270- router.ServeHTTP(responseRecorder, request)
271-
272- select {
273- case <-ch:
274- return
275- case <-time.After(time.Second * 1):
276- t.Error("didnt receive analytics event within time limit")
277- }
278-}
279-
280241 type ImageStorageMemory struct {
281242 *storage.StorageMemory
282243 Opts *storage.ImgProcessOpts
......@@ -337,8 +298,7 @@ func TestImageManipulation(t *testing.T) {
337298 Ratio: &storage.Ratio{},
338299 },
339300 }
340- ch := make(chan *db.AnalyticsVisits, 100)
341- router := NewWebRouter(cfg, cfg.Logger, tc.dbpool, st, ch)
301+ router := NewWebRouter(cfg, cfg.Logger, tc.dbpool, st)
342302 router.ServeHTTP(responseRecorder, request)
343303
344304 if responseRecorder.Code != tc.status {
+3 -43 prose/api.go #
......@@ -2,7 +2,6 @@ package prose
22
33 import (
44 "bytes"
5- "errors"
65 "fmt"
76 "html/template"
87 "net/http"
......@@ -89,11 +88,6 @@ type PostPageData struct {
8988 Diff template.HTML
9089 }
9190
92-type TransparencyPageData struct {
93- Site shared.SitePageData
94- Analytics *db.Analytics
95-}
96-
9791 type HeaderTxt struct {
9892 Title string
9993 Bio string
......@@ -270,21 +264,6 @@ func blogHandler(w http.ResponseWriter, r *http.Request) {
270264 postCollection = append(postCollection, p)
271265 }
272266
273- // track visit
274- ch := shared.GetAnalyticsQueue(r)
275- view, err := shared.AnalyticsVisitFromRequest(r, dbpool, user.ID)
276- if err == nil {
277- select {
278- case ch <- view:
279- default:
280- logger.Error("could not send analytics view to channel", "view", view)
281- }
282- } else {
283- if !errors.Is(err, shared.ErrAnalyticsDisabled) {
284- logger.Error("could not record analytics view", "err", err, "view", view)
285- }
286- }
287-
288267 data := BlogPageData{
289268 Site: *cfg.GetSiteData(),
290269 PageTitle: headerTxt.Title,
......@@ -350,7 +329,6 @@ func postHandler(w http.ResponseWriter, r *http.Request) {
350329 username := shared.GetUsernameFromRequest(r)
351330 subdomain := shared.GetSubdomain(r)
352331 cfg := shared.GetCfg(r)
353- ch := shared.GetAnalyticsQueue(r)
354332
355333 var slug string
356334 if !cfg.IsSubdomains() || subdomain == "" {
......@@ -429,21 +407,6 @@ func postHandler(w http.ResponseWriter, r *http.Request) {
429407 ogImageCard = parsedText.ImageCard
430408 }
431409
432- // track visit
433- view, err := shared.AnalyticsVisitFromRequest(r, dbpool, user.ID)
434- if err == nil {
435- view.PostID = post.ID
436- select {
437- case ch <- view:
438- default:
439- logger.Error("could not send analytics view to channel", "view", view)
440- }
441- } else {
442- if !errors.Is(err, shared.ErrAnalyticsDisabled) {
443- logger.Error("could not record analytics view", "err", err, "view", view)
444- }
445- }
446-
447410 unlisted := false
448411 if post.Hidden || post.PublishAt.After(time.Now()) {
449412 unlisted = true
......@@ -953,13 +916,10 @@ func StartApiServer() {
953916 mainRoutes := createMainRoutes(staticRoutes)
954917 subdomainRoutes := createSubdomainRoutes(staticRoutes)
955918
956- ch := make(chan *db.AnalyticsVisits, 100)
957- go shared.AnalyticsCollect(ch, dbpool, logger)
958919 apiConfig := &shared.ApiConfig{
959- Cfg: cfg,
960- Dbpool: dbpool,
961- Storage: st,
962- AnalyticsQueue: ch,
920+ Cfg: cfg,
921+ Dbpool: dbpool,
922+ Storage: st,
963923 }
964924 handler := shared.CreateServe(mainRoutes, subdomainRoutes, apiConfig)
965925 router := http.HandlerFunc(handler)
+17 -0 shared/api.go #
......@@ -13,6 +13,23 @@ import (
1313 "github.com/picosh/utils"
1414 )
1515
16+type SubdomainProps struct {
17+ ProjectName string
18+ Username string
19+}
20+
21+func GetProjectFromSubdomain(subdomain string) (*SubdomainProps, error) {
22+ props := &SubdomainProps{}
23+ strs := strings.SplitN(subdomain, "-", 2)
24+ props.Username = strs[0]
25+ if len(strs) == 2 {
26+ props.ProjectName = strs[1]
27+ } else {
28+ props.ProjectName = props.Username
29+ }
30+ return props, nil
31+}
32+
1633 func CorsHeaders(headers http.Header) {
1734 headers.Add("Access-Control-Allow-Origin", "*")
1835 headers.Add("Vary", "Origin")
+3 -10 shared/router.go #
......@@ -69,10 +69,9 @@ func CreatePProfRoutesMux(mux *http.ServeMux) {
6969 }
7070
7171 type ApiConfig struct {
72- Cfg *ConfigSite
73- Dbpool db.DB
74- Storage storage.StorageServe
75- AnalyticsQueue chan *db.AnalyticsVisits
72+ Cfg *ConfigSite
73+ Dbpool db.DB
74+ Storage storage.StorageServe
7675 }
7776
7877 func (hc *ApiConfig) HasPrivilegedAccess(apiToken string) bool {
......@@ -93,7 +92,6 @@ func (hc *ApiConfig) CreateCtx(prevCtx context.Context, subdomain string) contex
9392 ctx = context.WithValue(ctx, ctxDBKey{}, hc.Dbpool)
9493 ctx = context.WithValue(ctx, ctxStorageKey{}, hc.Storage)
9594 ctx = context.WithValue(ctx, ctxCfg{}, hc.Cfg)
96- ctx = context.WithValue(ctx, ctxAnalyticsQueue{}, hc.AnalyticsQueue)
9795 return ctx
9896 }
9997
......@@ -172,7 +170,6 @@ type ctxDBKey struct{}
172170 type ctxStorageKey struct{}
173171 type ctxLoggerKey struct{}
174172 type ctxCfg struct{}
175-type ctxAnalyticsQueue struct{}
176173
177174 type CtxSubdomainKey struct{}
178175 type ctxKey struct{}
......@@ -228,10 +225,6 @@ func GetCustomDomain(host string, space string) string {
228225 return ""
229226 }
230227
231-func GetAnalyticsQueue(r *http.Request) chan *db.AnalyticsVisits {
232- return r.Context().Value(ctxAnalyticsQueue{}).(chan *db.AnalyticsVisits)
233-}
234-
235228 func GetApiToken(r *http.Request) string {
236229 authHeader := r.Header.Get("authorization")
237230 if authHeader == "" {
+1 -0 sql/migrations/20241125_add_content_type_to_analytics.sql #
......@@ -0,0 +1,1 @@
1+ALTER TABLE analytics_visits ADD COLUMN content_type varchar(256);
+0 -0 test.txt # 
Binaries are not rendered as diffs.
Back to top