AES-XTS VFS (#171)

Co-authored-by: Ben Krieger <[email protected]>

Author: ncruces

vfs/adiantum/math.go renamed to internal/util/math.go

@@ -1,4 +1,4 @@
-package adiantum
+package util
 
 func abs(n int) int {
 	if n < 0 {
@@ -7,16 +7,16 @@ func abs(n int) int {
 	return n
 }
 
-func gcd(m, n int) int {
+func GCD(m, n int) int {
 	for n != 0 {
 		m, n = n, m%n
 	}
 	return abs(m)
 }
 
-func lcm(m, n int) int {
+func LCM(m, n int) int {
 	if n == 0 {
 		return 0
 	}
-	return abs(n) * (abs(m) / gcd(m, n))
+	return abs(n) * (abs(m) / GCD(m, n))
 }

vfs/adiantum/math_test.go renamed to internal/util/math_test.go

@@ -1,4 +1,4 @@
-package adiantum
+package util
 
 import (
 	"math"
@@ -25,7 +25,7 @@ func Test_abs(t *testing.T) {
 	}
 }
 
-func Test_gcd(t *testing.T) {
+func Test_GCD(t *testing.T) {
 	tests := []struct {
 		arg1 int
 		arg2 int
@@ -46,14 +46,14 @@ func Test_gcd(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run("", func(t *testing.T) {
-			if got := gcd(tt.arg1, tt.arg2); got != tt.want {
+			if got := GCD(tt.arg1, tt.arg2); got != tt.want {
 				t.Errorf("gcd(%d, %d) = %d, want %d", tt.arg1, tt.arg2, got, tt.want)
 			}
 		})
 	}
 }
 
-func Test_lcm(t *testing.T) {
+func Test_LCM(t *testing.T) {
 	tests := []struct {
 		arg1 int
 		arg2 int
@@ -74,7 +74,7 @@ func Test_lcm(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run("", func(t *testing.T) {
-			if got := lcm(tt.arg1, tt.arg2); got != tt.want {
+			if got := LCM(tt.arg1, tt.arg2); got != tt.want {
 				t.Errorf("lcm(%d, %d) = %d, want %d", tt.arg1, tt.arg2, got, tt.want)
 			}
 		})

vfs/adiantum/README.md

@@ -11,7 +11,7 @@ The default Adiantum construction uses XChaCha12 for its stream cipher,
 AES for its block cipher, and NH and Poly1305 for hashing.\
 Additionally, we use [Argon2id](https://pkg.go.dev/golang.org/x/crypto/argon2#hdr-Argon2id)
 to derive 256-bit keys from plain text where needed.
-File contents are encrypted in 4K blocks, matching the
+File contents are encrypted in 4 KiB blocks, matching the
 [default](https://sqlite.org/pgszchng2016.html) SQLite page size.
 
 The VFS encrypts all files _except_
@@ -53,6 +53,10 @@ and want to protect against forgery, you should sign your backups,
 and verify signatures before restoring them.
 
 This is slightly weaker than other forms of SQLite encryption
-that include block-level [MACs](https://en.wikipedia.org/wiki/Message_authentication_code).
-Block-level MACs can protect against forging individual blocks,
+that include page-level [MACs](https://en.wikipedia.org/wiki/Message_authentication_code).
+Page-level MACs can protect against forging individual pages,
 but can't prevent them from being reverted to former versions of themselves.
+
+> [!TIP]
+> The [`"xts"`](../xts/README.md) package also offers encryption at rest.
+> AES-XTS uses _only_ NIST and FIPS-140 approved cryptographic primitives.

vfs/adiantum/adiantum_test.go

@@ -42,6 +42,11 @@ func Test_fileformat(t *testing.T) {
 	if version != 0xBADDB {
 		t.Error(version)
 	}
+
+	_, err = db.Exec(`PRAGMA integrity_check`)
+	if err != nil {
+		t.Error(err)
+	}
 }
 
 func Benchmark_nokey(b *testing.B) {
@@ -57,6 +62,7 @@ func Benchmark_nokey(b *testing.B) {
 		db.Close()
 	}
 }
+
 func Benchmark_hexkey(b *testing.B) {
 	tmp := filepath.Join(b.TempDir(), "test.db")
 	sqlite3.Initialize()

vfs/adiantum/api.go

@@ -45,13 +45,17 @@ func init() {
 // Register registers an encrypting VFS, wrapping a base VFS,
 // and possibly using a custom HBSH cipher construction.
 // To use the default Adiantum construction, set cipher to nil.
+//
+// The default construction uses a 32 byte key/hexkey.
+// If a textkey is provided, the default KDF is Argon2id
+// with 64 MiB of memory, 3 iterations, and 4 threads.
 func Register(name string, base vfs.VFS, cipher HBSHCreator) {
 	if cipher == nil {
 		cipher = adiantumCreator{}
 	}
 	vfs.Register(name, &hbshVFS{
 		VFS:  base,
-		hbsh: cipher,
+		init: cipher,
 	})
 }
 

vfs/adiantum/hbsh.go

@@ -13,7 +13,7 @@ import (
 
 type hbshVFS struct {
 	vfs.VFS
-	hbsh HBSHCreator
+	init HBSHCreator
 }
 
 func (h *hbshVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
@@ -39,35 +39,40 @@ func (h *hbshVFS) OpenFilename(name *vfs.Filename, flags vfs.OpenFlag) (file vfs
 	} else {
 		var key []byte
 		if params := name.URIParameters(); name == nil {
-			key = h.hbsh.KDF("") // Temporary files get a random key.
+			key = h.init.KDF("") // Temporary files get a random key.
 		} else if t, ok := params["key"]; ok {
 			key = []byte(t[0])
 		} else if t, ok := params["hexkey"]; ok {
 			key, _ = hex.DecodeString(t[0])
-		} else if t, ok := params["textkey"]; ok {
-			key = h.hbsh.KDF(t[0])
+		} else if t, ok := params["textkey"]; ok && len(t[0]) > 0 {
+			key = h.init.KDF(t[0])
 		} else if flags&vfs.OPEN_MAIN_DB != 0 {
 			// Main datatabases may have their key specified as a PRAGMA.
-			return &hbshFile{File: file, reset: h.hbsh}, flags, nil
+			return &hbshFile{File: file, init: h.init}, flags, nil
 		}
-		hbsh = h.hbsh.HBSH(key)
+		hbsh = h.init.HBSH(key)
 	}
 
 	if hbsh == nil {
 		return nil, flags, sqlite3.CANTOPEN
 	}
-	return &hbshFile{File: file, hbsh: hbsh, reset: h.hbsh}, flags, nil
+	return &hbshFile{File: file, hbsh: hbsh, init: h.init}, flags, nil
 }
 
+// Larger blocks improve both security (wide-block cipher)
+// and throughput (cheap hashes amortize the block cipher's cost).
+// Use the default SQLite page size;
+// smaller pages pay the cost of unaligned access.
+// https://sqlite.org/pgszchng2016.html
 const (
 	tweakSize = 8
 	blockSize = 4096
 )
 
 type hbshFile struct {
 	vfs.File
+	init  HBSHCreator
 	hbsh  *hbsh.HBSH
-	reset HBSHCreator
 	tweak [tweakSize]byte
 	block [blockSize]byte
 }
@@ -80,15 +85,17 @@ func (h *hbshFile) Pragma(name string, value string) (string, error) {
 	case "hexkey":
 		key, _ = hex.DecodeString(value)
 	case "textkey":
-		key = h.reset.KDF(value)
+		if len(value) > 0 {
+			key = h.init.KDF(value)
+		}
 	default:
 		if f, ok := h.File.(vfs.FilePragma); ok {
 			return f.Pragma(name, value)
 		}
 		return "", sqlite3.NOTFOUND
 	}
 
-	if h.hbsh = h.reset.HBSH(key); h.hbsh != nil {
+	if h.hbsh = h.init.HBSH(key); h.hbsh != nil {
 		return "ok", nil
 	}
 	return "", sqlite3.CANTOPEN
@@ -99,7 +106,7 @@ func (h *hbshFile) ReadAt(p []byte, off int64) (n int, err error) {
 		// Only OPEN_MAIN_DB can have a missing key.
 		if off == 0 && len(p) == 100 {
 			// SQLite is trying to read the header of a database file.
-			// Pretend the file is empty so the key may specified as a PRAGMA.
+			// Pretend the file is empty so the key may be specified as a PRAGMA.
 			return 0, io.EOF
 		}
 		return 0, sqlite3.CANTOPEN
@@ -187,7 +194,7 @@ func (h *hbshFile) Truncate(size int64) error {
 }
 
 func (h *hbshFile) SectorSize() int {
-	return lcm(h.File.SectorSize(), blockSize)
+	return util.LCM(h.File.SectorSize(), blockSize)
 }
 
 func (h *hbshFile) DeviceCharacteristics() vfs.DeviceCharacteristic {

vfs/adiantum/testdata/test.db

@@ -21,6 +21,7 @@ import (
 	"github.com/ncruces/go-sqlite3/vfs"
 	_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
 	"github.com/ncruces/go-sqlite3/vfs/memdb"
+	_ "github.com/ncruces/go-sqlite3/vfs/xts"
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/api"
 	"github.com/tetratelabs/wazero/experimental"
@@ -293,6 +294,52 @@ func Test_crash01_adiantum_wal(t *testing.T) {
 	mod.Close(ctx)
 }
 
+func Test_crash01_xts(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+	if os.Getenv("CI") != "" {
+		t.Skip("skipping in CI")
+	}
+	if !vfs.SupportsFileLocking {
+		t.Skip("skipping without locks")
+	}
+
+	ctx := util.NewContext(newContext(t))
+	name := "file:" + filepath.Join(t.TempDir(), "test.db") +
+		"?hexkey=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	cfg := config(ctx).WithArgs("mptest", name, "crash01.test",
+		"--vfs", "xts")
+	mod, err := rt.InstantiateModule(ctx, module, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	mod.Close(ctx)
+}
+
+func Test_crash01_xts_wal(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+	if os.Getenv("CI") != "" {
+		t.Skip("skipping in CI")
+	}
+	if !vfs.SupportsSharedMemory {
+		t.Skip("skipping without shared memory")
+	}
+
+	ctx := util.NewContext(newContext(t))
+	name := "file:" + filepath.Join(t.TempDir(), "test.db") +
+		"?hexkey=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	cfg := config(ctx).WithArgs("mptest", name, "crash01.test",
+		"--vfs", "xts", "--journalmode", "wal")
+	mod, err := rt.InstantiateModule(ctx, module, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	mod.Close(ctx)
+}
+
 func newContext(t *testing.T) context.Context {
 	return context.WithValue(context.Background(), logger{}, &testWriter{T: t})
 }

vfs/tests/mptest/mptest_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ncruces/go-sqlite3/vfs"
 	_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
 	_ "github.com/ncruces/go-sqlite3/vfs/memdb"
+	_ "github.com/ncruces/go-sqlite3/vfs/xts"
 )
 
 //go:embed testdata/speedtest1.wasm.bz2
@@ -126,3 +127,22 @@ func Benchmark_adiantum(b *testing.B) {
 	}
 	mod.Close(ctx)
 }
+
+func Benchmark_xts(b *testing.B) {
+	output.Reset()
+	ctx := util.NewContext(context.Background())
+	name := "file:" + filepath.Join(b.TempDir(), "test.db") +
+		"?hexkey=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	args := append(options, "--vfs", "xts", "--size", strconv.Itoa(b.N), name)
+	cfg := wazero.NewModuleConfig().
+		WithArgs(args...).WithName("speedtest1").
+		WithStdout(&output).WithStderr(&output).
+		WithSysWalltime().WithSysNanotime().WithSysNanosleep().
+		WithOsyield(runtime.Gosched).
+		WithRandSource(rand.Reader)
+	mod, err := rt.InstantiateModule(ctx, module, cfg)
+	if err != nil {
+		b.Fatal(err)
+	}
+	mod.Close(ctx)
+}

vfs/tests/speedtest1/speedtest1_test.go

@@ -0,0 +1,63 @@
+# Go `xts` SQLite VFS
+
+This package wraps an SQLite VFS to offer encryption at rest.
+
+The `"xts"` VFS wraps the default SQLite VFS using the
+[AES-XTS](https://pkg.go.dev/golang.org/x/crypto/xts)
+tweakable and length-preserving encryption.\
+In general, any XTS construction can be used to wrap any VFS.
+
+The default AES-XTS construction uses AES-128, AES-192, or AES-256
+for its block cipher.
+Additionally, we use [PBKDF2-HMAC-SHA512](https://pkg.go.dev/golang.org/x/crypto/pbkdf2)
+to derive AES-128 keys from plain text where needed.
+File contents are encrypted in 512 byte sectors, matching the
+[minimum](https://sqlite.org/fileformat.html#pages) SQLite page size.
+
+The VFS encrypts all files _except_
+[super journals](https://sqlite.org/tempfiles.html#super_journal_files):
+these _never_ contain database data, only filenames,
+and padding them to the sector size is problematic.
+Temporary files _are_ encrypted with **random** AES-128 keys,
+as they _may_ contain database data.
+To avoid the overhead of encrypting temporary files,
+keep them in memory:
+
+    PRAGMA temp_store = memory;
+
+> [!IMPORTANT]
+> XTS is a cipher mode typically used for disk encryption.
+> The standard threat model for disk encryption considers an adversary
+> that can read multiple snapshots of a disk.
+> The only security property that disk encryption provides
+> is that all information such an adversary can obtain
+> is whether the data in a sector has or has not changed over time.
+
+The encryption offered by this package is fully deterministic.
+
+This means that an adversary who can get ahold of multiple snapshots
+(e.g. backups) of a database file can learn precisely:
+which sectors changed, which ones didn't, which got reverted.
+
+This is slightly weaker than other forms of SQLite encryption
+that include *some* nondeterminism; with limited nondeterminism,
+an adversary can't distinguish between
+sectors that actually changed, and sectors that got reverted.
+
+> [!CAUTION]
+> This package does not claim protect databases against tampering or forgery.
+
+The major practical consequence of the above point is that,
+if you're keeping `"xts"` encrypted backups of your database,
+and want to protect against forgery, you should sign your backups,
+and verify signatures before restoring them.
+
+This is slightly weaker than other forms of SQLite encryption
+that include page-level [MACs](https://en.wikipedia.org/wiki/Message_authentication_code).
+Page-level MACs can protect against forging individual pages,
+but can't prevent them from being reverted to former versions of themselves.
+
+> [!TIP]
+> The [`"adiantum"`](../adiantum/README.md) package also offers encryption at rest.
+> In general Adiantum performs significantly better,
+> and as a "wide-block" cipher, _may_ offer improved security.