chore: rsync test

Logs

erock created pr with ps-100 on 2025-02-21T21:07:49Z

erock added ps-101 on 2025-02-21T21:21:15Z

erock added ps-102 on 2025-02-21T21:21:56Z

erock added ps-103 on 2025-02-21T21:22:20Z

erock added ps-104 on 2025-02-22T02:12:03Z

erock changed status on 2025-03-13T14:53:20Z {"status":"accepted"}

Patchsets

ps-100 by erock on 2025-02-21T21:07:49Z

Range Diff ↕ rd-101


                1:  0306c3f < -:  ------- chore: rsync test


                2:  971b354 ! 1:  745b513 chore: add `rsync --delete` test

ps-101 by erock on 2025-02-21T21:21:15Z

Range Diff ↕ rd-102


                1:  745b513 ! 1:  76a6734 chore: add `rsync --delete` test

ps-102 by erock on 2025-02-21T21:21:56Z

Range Diff ↕ rd-103


                1:  76a6734 ! 1:  57bb60c chore: add `rsync --delete` test

ps-103 by erock on 2025-02-21T21:22:20Z

Range Diff ↕ rd-104


                1:  57bb60c = 1:  57bb60c chore: add `rsync --delete` test


                -:  ------- > 2:  5c8d929 chore: test is failing on `--delete`

ps-104 by erock on 2025-02-22T02:12:03Z

Patchset `ps-104`

chore: add `rsync --delete` test

Eric Bower

2025-02-17T15:36:03Z

pgs/ssh_test.go

+149 -5

chore: test is failing on `--delete`

Eric Bower

2025-02-22T02:10:34Z

go.mod

+1 -0

go.sum

+2 -0

pgs/ssh_test.go

+23 -14

chore: add `rsync --delete` test

I can't figure out how to format the `exec.Command` for rsync with `-e`
flag

Current error:

```
[rsync -rv -e "ssh -p 2222 -o IdentitiesOnly=yes -i /tmp/rsync-3246995037/id_ed25519 -o StrictHostKeyChecking=no" /tmp/rsync-3246995037/ localhost:/test]
rsync: [sender] Failed to exec ssh -p 2222 -o IdentitiesOnly=yes -i /tmp/rsync-3246995037/id_ed25519 -o StrictHostKeyChecking=no: No such file or directory (2)
rsync error: error in IPC code (code 14) at pipe.c(85) [sender=3.4.0]
rsync: connection unexpectedly closed (0 bytes received so far) [sender]
rsync error: error in IPC code (code 14) at io.c(232) [sender=3.4.0]
 exit status 14
--- FAIL: TestSshServerRsync (0.10s)
    ssh_test.go:152: exit status 14
```

pgs/ssh_test.go link

+149 -5

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196


diff --git a/pgs/ssh_test.go b/pgs/ssh_test.go
index d30fa31..fe97468 100644
--- a/pgs/ssh_test.go
+++ b/pgs/ssh_test.go
@@ -3,9 +3,14 @@ package pgs
 import (
 	"crypto/ed25519"
 	"crypto/rand"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
 	"io"
 	"log/slog"
 	"os"
+	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -18,7 +23,7 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
-func TestSshServer(t *testing.T) {
+func TestSshServerSftp(t *testing.T) {
 	logger := slog.Default()
 	dbpool := pgsdb.NewDBMemory(logger)
 	// setup test data
@@ -57,9 +62,142 @@ func TestSshServer(t *testing.T) {
 	done <- nil
 }
 
+func TestSshServerRsync(t *testing.T) {
+	logger := slog.Default()
+	dbpool := pgsdb.NewDBMemory(logger)
+	// setup test data
+	dbpool.SetupTestData()
+	st, err := storage.NewStorageMemory(map[string]map[string]string{})
+	if err != nil {
+		panic(err)
+	}
+	cfg := NewPgsConfig(logger, dbpool, st)
+	done := make(chan error)
+	go StartSshServer(cfg, done)
+	// Hack to wait for startup
+	time.Sleep(time.Millisecond * 100)
+
+	user := GenerateUser()
+	key := utils.KeyForKeyText(user.signer.PublicKey())
+	// add user's pubkey to the default test account
+	dbpool.Pubkeys = append(dbpool.Pubkeys, &db.PublicKey{
+		ID:     "nice-pubkey",
+		UserID: dbpool.Users[0].ID,
+		Key:    key,
+	})
+
+	conn, err := user.NewClient()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer conn.Close()
+
+	// open an SFTP session over an existing ssh connection.
+	client, err := sftp.NewClient(conn)
+	if err != nil {
+		cfg.Logger.Error("could not create sftp client", "err", err)
+		panic(err)
+	}
+	defer client.Close()
+
+	name, err := os.MkdirTemp("", "rsync-")
+	if err != nil {
+		panic(err)
+	}
+
+	// remove the temporary directory at the end of the program
+	defer os.RemoveAll(name)
+
+	block := &pem.Block{
+		Type:  "PRIVATE KEY",
+		Bytes: user.privateKey,
+	}
+	keyFile := filepath.Join(name, "id_ed25519")
+	err = os.WriteFile(
+		keyFile,
+		pem.EncodeToMemory(block), 0600,
+	)
+
+	index := "<!doctype html><html><body>index</body></html>"
+	err = os.WriteFile(
+		filepath.Join(name, "index.html"),
+		[]byte(index), 0666,
+	)
+
+	about := "<!doctype html><html><body>about</body></html>"
+	aboutFile := filepath.Join(name, "about.html")
+	err = os.WriteFile(
+		aboutFile,
+		[]byte(about), 0666,
+	)
+
+	contact := "<!doctype html><html><body>contact</body></html>"
+	err = os.WriteFile(
+		filepath.Join(name, "contact.html"),
+		[]byte(contact), 0666,
+	)
+
+	eCmd := fmt.Sprintf(
+		`"ssh -p 2222 -o IdentitiesOnly=yes -i %s -o StrictHostKeyChecking=no"`,
+		keyFile,
+	)
+
+	// copy files
+	cmd := exec.Command("rsync", "-rv", "-e", eCmd, name+"/", "localhost:/test")
+	fmt.Println(cmd.Args)
+	result, err := cmd.CombinedOutput()
+	if err != nil {
+		fmt.Println(string(result), err)
+		t.Error(err)
+		return
+	}
+
+	// check it's there
+	fi, err := client.Lstat("about.html")
+	if err != nil {
+		cfg.Logger.Error("could not get stat for file", "err", err)
+		t.Error("about.html not found")
+		return
+	}
+	if fi.Size() != 0 {
+		cfg.Logger.Error("about.html wrong size", "size", fi.Size())
+		t.Error("about.html wrong size")
+		return
+	}
+
+	// remove about file
+	os.Remove(aboutFile)
+
+	// copy files with delete
+	delCmd := exec.Command("rsync", "-rv", "--delete", "-e", eCmd, name+"/", "localhost:/test")
+	err = delCmd.Run()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	done <- nil
+}
+
+func createTmpFile(name, contents, ext string) *os.File {
+	file, err := os.CreateTemp("tmp", fmt.Sprintf("%s-*.%s", name, ext))
+	if err != nil {
+		panic(err)
+	}
+
+	data := []byte(contents)
+	if _, err := file.Write(data); err != nil {
+		panic(err)
+	}
+
+	return file
+}
+
 type UserSSH struct {
-	username string
-	signer   ssh.Signer
+	username   string
+	signer     ssh.Signer
+	privateKey []byte
 }
 
 func NewUserSSH(username string, signer ssh.Signer) *UserSSH {
@@ -146,14 +284,20 @@ func GenerateUser() UserSSH {
 		panic(err)
 	}
 
+	b, err := x509.MarshalPKCS8PrivateKey(userKey)
+	if err != nil {
+		panic(err)
+	}
+
 	userSigner, err := ssh.NewSignerFromKey(userKey)
 	if err != nil {
 		panic(err)
 	}
 
 	return UserSSH{
-		username: "testuser",
-		signer:   userSigner,
+		username:   "testuser",
+		signer:     userSigner,
+		privateKey: b,
 	}
 }
 

chore: test is failing on `--delete`

Alright, the test is failing because of the new `--delete` flag.

```
time=2025-02-21T21:10:11.225-05:00 level=ERROR source=/home/erock/go/pkg/mod/github.com/picosh/send@v0.0.0-20250213162645-ec2027b68462/protocols/rsync/rsync.go:157 msg="error running rsync middleware" service=pgs err="runtime error: index out of range [7876097] with length 0"
```

go.mod link

+1 -0

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12


diff --git a/go.mod b/go.mod
index ced7737..d93cede 100644
--- a/go.mod
+++ b/go.mod
@@ -48,6 +48,7 @@ require (
 	github.com/jmoiron/sqlx v1.4.0
 	github.com/lib/pq v1.10.9
 	github.com/microcosm-cc/bluemonday v1.0.27
+	github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a
 	github.com/minio/minio-go/v7 v7.0.80
 	github.com/mmcdole/gofeed v1.3.0
 	github.com/muesli/reflow v0.3.0

go.sum link

+2 -0

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13


diff --git a/go.sum b/go.sum
index 89d665e..5ce2828 100644
--- a/go.sum
+++ b/go.sum
@@ -634,6 +634,8 @@ github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKju
 github.com/miekg/dns v1.1.45/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=
 github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=
+github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a h1:eU8j/ClY2Ty3qdHnn0TyW3ivFoPC/0F1gQZz8yTxbbE=
+github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a/go.mod h1:v8eSC2SMp9/7FTKUncp7fH9IwPfw+ysMObcEz5FWheQ=
 github.com/minio/madmin-go/v3 v3.0.77 h1:cqp5kVeT5anDyocvoN81puwpy+GN7t+Xdj6xCLpnONE=
 github.com/minio/madmin-go/v3 v3.0.77/go.mod h1:ku/RUc2xeo4Uui/GHUURMoNEVXk4Ih40xA3KHLyMF1o=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=

pgs/ssh_test.go link

+23 -14

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112


diff --git a/pgs/ssh_test.go b/pgs/ssh_test.go
index fe97468..bad40a5 100644
--- a/pgs/ssh_test.go
+++ b/pgs/ssh_test.go
@@ -3,7 +3,6 @@ package pgs
 import (
 	"crypto/ed25519"
 	"crypto/rand"
-	"crypto/x509"
 	"encoding/pem"
 	"fmt"
 	"io"
@@ -15,6 +14,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/mikesmitty/edkey"
 	"github.com/picosh/pico/db"
 	pgsdb "github.com/picosh/pico/pgs/db"
 	"github.com/picosh/pico/shared/storage"
@@ -24,7 +24,13 @@ import (
 )
 
 func TestSshServerSftp(t *testing.T) {
-	logger := slog.Default()
+	opts := &slog.HandlerOptions{
+		AddSource: true,
+		Level:     slog.LevelInfo,
+	}
+	logger := slog.New(
+		slog.NewTextHandler(os.Stdout, opts),
+	)
 	dbpool := pgsdb.NewDBMemory(logger)
 	// setup test data
 	dbpool.SetupTestData()
@@ -63,7 +69,13 @@ func TestSshServerSftp(t *testing.T) {
 }
 
 func TestSshServerRsync(t *testing.T) {
-	logger := slog.Default()
+	opts := &slog.HandlerOptions{
+		AddSource: true,
+		Level:     slog.LevelInfo,
+	}
+	logger := slog.New(
+		slog.NewTextHandler(os.Stdout, opts),
+	)
 	dbpool := pgsdb.NewDBMemory(logger)
 	// setup test data
 	dbpool.SetupTestData()
@@ -107,10 +119,10 @@ func TestSshServerRsync(t *testing.T) {
 	}
 
 	// remove the temporary directory at the end of the program
-	defer os.RemoveAll(name)
+	// defer os.RemoveAll(name)
 
 	block := &pem.Block{
-		Type:  "PRIVATE KEY",
+		Type:  "OPENSSH PRIVATE KEY",
 		Bytes: user.privateKey,
 	}
 	keyFile := filepath.Join(name, "id_ed25519")
@@ -139,7 +151,7 @@ func TestSshServerRsync(t *testing.T) {
 	)
 
 	eCmd := fmt.Sprintf(
-		`"ssh -p 2222 -o IdentitiesOnly=yes -i %s -o StrictHostKeyChecking=no"`,
+		"ssh -p 2222 -o IdentitiesOnly=yes -i %s -o StrictHostKeyChecking=no",
 		keyFile,
 	)
 
@@ -154,13 +166,13 @@ func TestSshServerRsync(t *testing.T) {
 	}
 
 	// check it's there
-	fi, err := client.Lstat("about.html")
+	fi, err := client.Lstat("/test/about.html")
 	if err != nil {
 		cfg.Logger.Error("could not get stat for file", "err", err)
 		t.Error("about.html not found")
 		return
 	}
-	if fi.Size() != 0 {
+	if fi.Size() != 46 {
 		cfg.Logger.Error("about.html wrong size", "size", fi.Size())
 		t.Error("about.html wrong size")
 		return
@@ -171,8 +183,9 @@ func TestSshServerRsync(t *testing.T) {
 
 	// copy files with delete
 	delCmd := exec.Command("rsync", "-rv", "--delete", "-e", eCmd, name+"/", "localhost:/test")
-	err = delCmd.Run()
+	result, err = delCmd.CombinedOutput()
 	if err != nil {
+		fmt.Println(string(result), err)
 		t.Error(err)
 		return
 	}
@@ -284,11 +297,7 @@ func GenerateUser() UserSSH {
 		panic(err)
 	}
 
-	b, err := x509.MarshalPKCS8PrivateKey(userKey)
-	if err != nil {
-		panic(err)
-	}
-
+	b := edkey.MarshalED25519PrivateKey(userKey)
 	userSigner, err := ssh.NewSignerFromKey(userKey)
 	if err != nil {
 		panic(err)

Logs

Patchsets

Patchset ps-104

chore: add `rsync --delete` test

chore: test is failing on `--delete`

chore: add `rsync --delete` test

chore: test is failing on `--delete`

Patchset `ps-104`