Remove libolm dependency (matrix-org#738)

* Remove libolm dependency

See matrix-org#729 for more information.

This is a breaking change because it removes blueprint functionality.
Specifically, it removes:
 - the field `OneTimeKeys` from the `User` struct.
 - the performance blueprint `BlueprintPerfE2EERoom`.

* Remove olm dep in client.go

* Fix device list test

Author: kegsay

.github/workflows/ci.yaml

@@ -18,9 +18,6 @@ jobs:
       - uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
-      - name: "Install Complement Dependencies"
-        run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
       - name: "Run internal Complement tests"
         run: |
           go test ./internal/...
@@ -69,7 +66,6 @@ jobs:
       # servers which listen on random high numbered ports.
       - name: "Install Complement Dependencies"
         run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
           go install -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
           mkdir .gotestfmt/github -p
           cp .ci/complement_package.gotpl .gotestfmt/github/package.gotpl

b/blueprints.go

@@ -32,7 +32,6 @@ var KnownBlueprints = map[string]*Blueprint{
 	BlueprintOneToOneRoom.Name:                &BlueprintOneToOneRoom,
 	BlueprintPerfManyMessages.Name:            &BlueprintPerfManyMessages,
 	BlueprintPerfManyRooms.Name:               &BlueprintPerfManyRooms,
-	BlueprintPerfE2EERoom.Name:                &BlueprintPerfE2EERoom,
 }
 
 // Blueprint represents an entire deployment to make.
@@ -64,10 +63,6 @@ type User struct {
 	AvatarURL   string
 	AccountData []AccountData
 	DeviceID    *string
-	// Enable end-to end encryption for this user and upload the given
-	// amount of one-time keys. This requires the DeviceId to be set as
-	// well.
-	OneTimeKeys uint
 }
 
 type AccountData struct {

b/perf_e2ee_room.go

@@ -3,9 +3,12 @@ package client
 import (
 	"bytes"
 	"context" // nolint:gosec
+	"crypto/ed25519"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
+	"math/rand"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -16,7 +19,7 @@ import (
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/tidwall/gjson"
-	"maunium.net/go/mautrix/crypto/olm"
+	"golang.org/x/crypto/curve25519"
 
 	"github.com/matrix-org/complement/b"
 	"github.com/matrix-org/complement/ct"
@@ -28,6 +31,12 @@ const (
 	CtxKeyWithRetryUntil ctxKey = "complement_retry_until" // contains *retryUntilParams
 )
 
+var (
+	// use a deterministic seed but globally so we don't generate the same numbers for each client.
+	// This could be non-deterministic if used concurrently.
+	prng = rand.New(rand.NewSource(42))
+)
+
 type retryUntilParams struct {
 	timeout time.Duration
 	untilFn func(*http.Response) bool
@@ -403,10 +412,21 @@ func (c *CSAPI) MustUploadKeys(t ct.TestLike, deviceKeys map[string]interface{},
 	return s.OTKCounts
 }
 
+// Generate realistic looking device keys and OTKs. They are not guaranteed to be 100% valid, but should
+// pass most server-side checks. Critically, these keys are generated using a Pseudo-Random Number Generator (PRNG)
+// for determinism and hence ARE NOT SECURE. DO NOT USE THIS OUTSIDE OF TESTS.
 func (c *CSAPI) MustGenerateOneTimeKeys(t ct.TestLike, otkCount uint) (deviceKeys map[string]interface{}, oneTimeKeys map[string]interface{}) {
 	t.Helper()
-	account := olm.NewAccount()
-	ed25519Key, curveKey := account.IdentityKeys()
+	ed25519PubKey, ed25519PrivKey, err := ed25519.GenerateKey(prng)
+	if err != nil {
+		ct.Fatalf(t, "failed to generate ed25519 key: %s", err)
+	}
+
+	curveKey := make([]byte, 32)
+	_, err = prng.Read(curveKey)
+	if err != nil {
+		ct.Fatalf(t, "failed to read from prng: %s", err)
+	}
 
 	ed25519KeyID := fmt.Sprintf("ed25519:%s", c.DeviceID)
 	curveKeyID := fmt.Sprintf("curve25519:%s", c.DeviceID)
@@ -416,38 +436,59 @@ func (c *CSAPI) MustGenerateOneTimeKeys(t ct.TestLike, otkCount uint) (deviceKey
 		"device_id":  c.DeviceID,
 		"algorithms": []interface{}{"m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"},
 		"keys": map[string]interface{}{
-			ed25519KeyID: ed25519Key.String(),
-			curveKeyID:   curveKey.String(),
+			ed25519KeyID: base64.RawStdEncoding.EncodeToString(ed25519PubKey),
+			curveKeyID:   base64.RawStdEncoding.EncodeToString(curveKey),
 		},
 	}
 
-	signature, _ := account.SignJSON(deviceKeys)
+	signJSON := func(input any) []byte {
+		inputJSON, err := json.Marshal(input)
+		if err != nil {
+			ct.Fatalf(t, "failed to marshal struct: %s", err)
+		}
+		inputJSON, err = gomatrixserverlib.CanonicalJSON(inputJSON)
+		if err != nil {
+			ct.Fatalf(t, "failed to canonical json: %s", err)
+		}
+		signature := ed25519.Sign(ed25519PrivKey, inputJSON)
+		if err != nil {
+			ct.Fatalf(t, "failed to sign json: %s", err)
+		}
+		return signature
+	}
 
 	deviceKeys["signatures"] = map[string]interface{}{
 		c.UserID: map[string]interface{}{
-			ed25519KeyID: signature,
+			ed25519KeyID: base64.RawStdEncoding.EncodeToString(signJSON(deviceKeys)),
 		},
 	}
-
-	account.GenOneTimeKeys(otkCount)
 	oneTimeKeys = map[string]interface{}{}
 
-	for kid, key := range account.OneTimeKeys() {
+	for i := uint(0); i < otkCount; i++ {
+		privateKeyBytes := make([]byte, 32)
+		_, err = prng.Read(privateKeyBytes)
+		if err != nil {
+			ct.Fatalf(t, "failed to read from prng", err)
+		}
+		key, err := curve25519.X25519(privateKeyBytes, curve25519.Basepoint)
+		if err != nil {
+			ct.Fatalf(t, "failed to generate curve pubkey: %s", err)
+		}
+		kid := fmt.Sprintf("%d", i)
 		keyID := fmt.Sprintf("signed_curve25519:%s", kid)
 		keyMap := map[string]interface{}{
-			"key": key.String(),
+			"key": base64.RawStdEncoding.EncodeToString(key),
 		}
 
-		signature, _ = account.SignJSON(keyMap)
-
 		keyMap["signatures"] = map[string]interface{}{
 			c.UserID: map[string]interface{}{
-				ed25519KeyID: signature,
+				ed25519KeyID: base64.RawStdEncoding.EncodeToString(signJSON(keyMap)),
 			},
 		}
 
 		oneTimeKeys[keyID] = keyMap
 	}
+
 	return deviceKeys, oneTimeKeys
 }
 

client/client.go

@@ -40,10 +40,6 @@ func ConvertToBlueprint(s *Snapshot, serverName string) (*b.Blueprint, error) {
 				DisplayName: local,
 				DeviceID:    &devices[i],
 			}
-			// set up OTKs if this user+device sends E2E messages
-			if devices[i] != NoEncryptedDevice {
-				user.OneTimeKeys = 100
-			}
 			// set DM list if this is the syncing user
 			if userID == s.UserID {
 				user.AccountData = append(user.AccountData, b.AccountData{

cmd/account-snapshot/internal/blueprint.go

@@ -14,7 +14,6 @@ require (
 	github.com/tidwall/sjson v1.2.5
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 	gonum.org/v1/plot v0.11.0
-	maunium.net/go/mautrix v0.11.0
 )
 
 require (

go.mod

@@ -16,7 +16,6 @@ import (
 	"time"
 
 	"github.com/tidwall/gjson"
-	"maunium.net/go/mautrix/crypto/olm"
 
 	"github.com/matrix-org/complement/b"
 )
@@ -418,9 +417,6 @@ func calculateUserInstructionSets(r *Runner, hs b.Homeserver) [][]instruction {
 		}
 		createdUsers[user.Localpart] = true
 
-		if user.OneTimeKeys > 0 {
-			instrs = append(instrs, instructionOneTimeKeyUpload(hs, user))
-		}
 		sets[i] = instrs
 	}
 	return sets
@@ -636,65 +632,6 @@ func instructionLogin(hs b.Homeserver, user b.User) instruction {
 	}
 }
 
-func instructionOneTimeKeyUpload(hs b.Homeserver, user b.User) instruction {
-	account := olm.NewAccount()
-	ed25519Key, curveKey := account.IdentityKeys()
-
-	userID := fmt.Sprintf("@%s:%s", user.Localpart, hs.Name)
-	deviceID := *user.DeviceID
-
-	ed25519KeyID := fmt.Sprintf("ed25519:%s", deviceID)
-	curveKeyID := fmt.Sprintf("curve25519:%s", deviceID)
-
-	deviceKeys := map[string]interface{}{
-		"user_id":    userID,
-		"device_id":  deviceID,
-		"algorithms": []string{"m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"},
-		"keys": map[string]string{
-			ed25519KeyID: ed25519Key.String(),
-			curveKeyID:   curveKey.String(),
-		},
-	}
-
-	signature, _ := account.SignJSON(deviceKeys)
-
-	deviceKeys["signatures"] = map[string]map[string]string{
-		userID: {
-			ed25519KeyID: signature,
-		},
-	}
-
-	account.GenOneTimeKeys(user.OneTimeKeys)
-
-	oneTimeKeys := map[string]interface{}{}
-
-	for kid, key := range account.OneTimeKeys() {
-		keyID := fmt.Sprintf("signed_curve25519:%s", kid)
-		keyMap := map[string]interface{}{
-			"key": key.String(),
-		}
-
-		signature, _ = account.SignJSON(keyMap)
-
-		keyMap["signatures"] = map[string]interface{}{
-			userID: map[string]string{
-				ed25519KeyID: signature,
-			},
-		}
-
-		oneTimeKeys[keyID] = keyMap
-	}
-	return instruction{
-		method:      "POST",
-		path:        "/_matrix/client/v3/keys/upload",
-		accessToken: fmt.Sprintf("user_@%s:%s", user.Localpart, hs.Name),
-		body: map[string]interface{}{
-			"device_keys":   deviceKeys,
-			"one_time_keys": oneTimeKeys,
-		},
-	}
-}
-
 // indexFor hashes the input and returns a number % numEntries
 func indexFor(input string, numEntries int) int {
 	hh := fnv.New32a()