Create Live Snapshot without shutting down node

arbitrum/recordingdb.go

@@ -41,6 +41,10 @@ func newRecordingKV(inner *triedb.Database, diskDb ethdb.KeyValueStore) *Recordi
 	return &RecordingKV{inner, diskDb, make(map[common.Hash][]byte), sync.Mutex{}, false}
 }
 
+func (db *RecordingKV) CreateDBSnapshot(dir string) error {
+	return errors.New("createDBSnapshot method is not supported")
+}
+
 func (db *RecordingKV) Has(key []byte) (bool, error) {
 	return false, errors.New("recording KV doesn't support Has")
 }

core/blockchain.go

@@ -1193,21 +1193,23 @@ func (bc *BlockChain) stopWithoutSaving() {
 	bc.wg.Wait()
 }
 
-// Stop stops the blockchain service. If any imports are currently in progress
-// it will abort them using the procInterrupt.
-func (bc *BlockChain) Stop() {
-	bc.stopWithoutSaving()
-
+func (bc *BlockChain) FlushToDisk(stopping bool) {
 	// Ensure that the entirety of the state snapshot is journaled to disk.
 	var snapBase common.Hash
 	if bc.snaps != nil {
 		var err error
 		if snapBase, err = bc.snaps.Journal(bc.CurrentBlock().Root); err != nil {
 			log.Error("Failed to journal state snapshot", "err", err)
 		}
-		bc.snaps.Release()
+		if stopping {
+			bc.snaps.Release()
+		}
 	}
 	if bc.triedb.Scheme() == rawdb.PathScheme {
+		// Called during live snapshotting
+		if !stopping {
+			return
+		}
 		// Ensure that the in-memory trie nodes are journaled to disk properly.
 		if err := bc.triedb.Journal(bc.CurrentBlock().Root); err != nil {
 			log.Info("Failed to journal in-memory trie nodes", "err", err)
@@ -1247,14 +1249,25 @@ func (bc *BlockChain) Stop() {
 					log.Error("Failed to commit recent state trie", "err", err)
 				}
 			}
-			for !bc.triegc.Empty() {
-				triedb.Dereference(bc.triegc.PopItem().Root)
-			}
-			if _, nodes, _ := triedb.Size(); nodes != 0 { // all memory is contained within the nodes return for hashdb
-				log.Error("Dangling trie nodes after full cleanup")
+			if stopping {
+				for !bc.triegc.Empty() {
+					triedb.Dereference(bc.triegc.PopItem().Root)
+				}
+				if _, nodes, _ := triedb.Size(); nodes != 0 { // all memory is contained within the nodes return for hashdb
+					log.Error("Dangling trie nodes after full cleanup")
+				}
 			}
 		}
 	}
+}
+
+// Stop stops the blockchain service. If any imports are currently in progress
+// it will abort them using the procInterrupt.
+func (bc *BlockChain) Stop() {
+	bc.stopWithoutSaving()
+
+	bc.FlushToDisk(true)
+
 	// Allow tracers to clean-up and release resources.
 	if bc.logger != nil && bc.logger.OnClose != nil {
 		bc.logger.OnClose()

core/rawdb/database.go

@@ -84,6 +84,16 @@ func (frdb *freezerdb) Freeze() error {
 	return nil
 }
 
+func (frdb *freezerdb) CreateDBSnapshot(dir string) error {
+	if err := frdb.KeyValueStore.CreateDBSnapshot(dir); err != nil {
+		return err
+	}
+	if err := frdb.AncientStore.CreateDBSnapshot(dir); err != nil {
+		return err
+	}
+	return nil
+}
+
 // nofreezedb is a database wrapper that disables freezer data retrievals.
 type nofreezedb struct {
 	ethdb.KeyValueStore
@@ -181,6 +191,16 @@ type dbWithWasmEntry struct {
 	wasmTargets  []ethdb.WasmTarget
 }
 
+func (db *dbWithWasmEntry) CreateDBSnapshot(dir string) error {
+	if err := db.Database.CreateDBSnapshot(dir); err != nil {
+		return err
+	}
+	if err := db.wasmDb.CreateDBSnapshot(dir); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (db *dbWithWasmEntry) WasmDataBase() (ethdb.KeyValueStore, uint32) {
 	return db.wasmDb, db.wasmCacheTag
 }

core/rawdb/freezer.go

@@ -19,6 +19,7 @@ package rawdb
 import (
 	"errors"
 	"fmt"
+	"io"
 	"math"
 	"os"
 	"path/filepath"
@@ -70,6 +71,8 @@ type Freezer struct {
 	tables       map[string]*freezerTable // Data tables for storing everything
 	instanceLock FileLock                 // File-system lock to prevent double opens
 	closeOnce    sync.Once
+
+	createSnapshotMutex sync.Mutex
 }
 
 // NewFreezer creates a freezer instance for maintaining immutable ordered
@@ -323,6 +326,8 @@ func (f *Freezer) TruncateTail(tail uint64) (uint64, error) {
 
 // Sync flushes all data tables to disk.
 func (f *Freezer) Sync() error {
+	f.createSnapshotMutex.Lock()
+	defer f.createSnapshotMutex.Unlock()
 	var errs []error
 	for _, table := range f.tables {
 		if err := table.Sync(); err != nil {
@@ -335,6 +340,63 @@ func (f *Freezer) Sync() error {
 	return nil
 }
 
+func (f *Freezer) CreateDBSnapshot(dir string) error {
+	f.createSnapshotMutex.Lock()
+	defer f.createSnapshotMutex.Unlock()
+	snapshotDir := filepath.Join(dir, "l2chaindata", "ancient", "chain") // Format currently used
+	if err := os.MkdirAll(snapshotDir, os.ModePerm); err != nil {
+		return fmt.Errorf("failed to create snapshot of ancient directory: %w", err)
+	}
+	// Manually copy contents of ancient to the snapshotDir, createSnapshotMutex makes sure ancient is not updated while copying contents
+	err := filepath.Walk(f.datadir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return fmt.Errorf("error accessing path %s: %w", path, err)
+		}
+		relPath, err := filepath.Rel(f.datadir, path)
+		if err != nil {
+			return fmt.Errorf("error calculating relative path: %w", err)
+		}
+		destPath := filepath.Join(snapshotDir, relPath)
+		if info.IsDir() {
+			if err := os.MkdirAll(destPath, info.Mode()); err != nil {
+				return fmt.Errorf("failed to create directory %s: %w", destPath, err)
+			}
+		} else {
+			if err := copyFile(path, destPath); err != nil {
+				return fmt.Errorf("failed to copy file %s: %w", path, err)
+			}
+		}
+		return nil
+	})
+	return err
+}
+
+func copyFile(src, dest string) error {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("failed to open source file: %w", err)
+	}
+	defer srcFile.Close()
+	destFile, err := os.Create(dest)
+	if err != nil {
+		return fmt.Errorf("failed to create destination file: %w", err)
+	}
+	defer destFile.Close()
+	_, err = io.Copy(destFile, srcFile)
+	if err != nil {
+		return fmt.Errorf("failed to copy contents: %w", err)
+	}
+	srcInfo, err := os.Stat(src)
+	if err != nil {
+		return fmt.Errorf("failed to stat source file: %w", err)
+	}
+	err = os.Chmod(dest, srcInfo.Mode())
+	if err != nil {
+		return fmt.Errorf("failed to set permissions: %w", err)
+	}
+	return nil
+}
+
 // validate checks that every table has the same boundary.
 // Used instead of `repair` in readonly mode.
 func (f *Freezer) validate() error {

core/rawdb/freezer_memory.go

@@ -230,6 +230,10 @@ func NewMemoryFreezer(readonly bool, tableName map[string]bool) *MemoryFreezer {
 	}
 }
 
+func (f *MemoryFreezer) CreateDBSnapshot(dir string) error {
+	return errors.New("createDBSnapshot method is not supported by MemoryFreezer")
+}
+
 // HasAncient returns an indicator whether the specified data exists.
 func (f *MemoryFreezer) HasAncient(kind string, number uint64) (bool, error) {
 	f.lock.RLock()

core/rawdb/freezer_resettable.go

@@ -17,6 +17,7 @@
 package rawdb
 
 import (
+	"errors"
 	"os"
 	"path/filepath"
 	"sync"
@@ -97,6 +98,10 @@ func (f *resettableFreezer) Reset() error {
 	return nil
 }
 
+func (f *resettableFreezer) CreateDBSnapshot(dir string) error {
+	return errors.New("createDBSnapshot method is not supported by resettableFreezer")
+}
+
 // Close terminates the chain freezer, unmapping all the data files.
 func (f *resettableFreezer) Close() error {
 	f.lock.RLock()

core/rawdb/table.go

@@ -17,6 +17,8 @@
 package rawdb
 
 import (
+	"errors"
+
 	"github.com/ethereum/go-ethereum/ethdb"
 )
 
@@ -35,6 +37,10 @@ func NewTable(db ethdb.Database, prefix string) ethdb.Database {
 	}
 }
 
+func (t *table) CreateDBSnapshot(dir string) error {
+	return errors.New("createDBSnapshot method is not supported")
+}
+
 // Close is a noop to implement the Database interface.
 func (t *table) Close() error {
 	return nil

ethdb/database.go

@@ -73,6 +73,9 @@ type KeyValueStore interface {
 	Iteratee
 	Compacter
 	io.Closer
+
+	// CreateDBSnapshot creates a copy of underlying database inside dir
+	CreateDBSnapshot(dir string) error
 }
 
 // AncientReaderOp contains the methods required to read from immutable ancient data.
@@ -169,6 +172,9 @@ type AncientStore interface {
 	AncientWriter
 	AncientStater
 	io.Closer
+
+	// CreateDBSnapshot creates a copy of underlying database inside dir
+	CreateDBSnapshot(dir string) error
 }
 
 type WasmTarget string

ethdb/leveldb/fake_leveldb.go

@@ -59,6 +59,10 @@ func configureOptions(customizeFn func(*opt.Options)) *opt.Options {
 }
 */
 
+func (db *Database) CreateDBSnapshot(dir string) error {
+	return errors.New("createDBSnapshot method is not supported by leveldb")
+}
+
 // Close stops the metrics collection, flushes any pending data to disk and closes
 // all io accesses to the underlying key-value store.
 func (db *Database) Close() error {

ethdb/leveldb/leveldb.go

@@ -166,6 +166,10 @@ func configureOptions(customizeFn func(*opt.Options)) *opt.Options {
 	return options
 }
 
+func (db *Database) CreateDBSnapshot(dir string) error {
+	return errors.New("createDBSnapshot method is not supported by leveldb")
+}
+
 // Close stops the metrics collection, flushes any pending data to disk and closes
 // all io accesses to the underlying key-value store.
 func (db *Database) Close() error {

ethdb/memorydb/memorydb.go

@@ -61,6 +61,10 @@ func NewWithCap(size int) *Database {
 	}
 }
 
+func (db *Database) CreateDBSnapshot(dir string) error {
+	return errors.New("createDBSnapshot method is not supported by memorydb")
+}
+
 // Close deallocates the internal map and ensures any consecutive data access op
 // fails with an error.
 func (db *Database) Close() error {

ethdb/pebble/pebble.go

@@ -22,6 +22,7 @@ package pebble
 import (
 	"bytes"
 	"fmt"
+	"path/filepath"
 	"runtime"
 	"sync"
 	"sync/atomic"
@@ -116,6 +117,11 @@ type Database struct {
 	writeOptions *pebble.WriteOptions
 }
 
+func (d *Database) CreateDBSnapshot(dir string) error {
+	snapshotDir := filepath.Join(dir, filepath.Base(d.Path()))
+	return d.db.Checkpoint(snapshotDir, pebble.WithFlushedWAL())
+}
+
 func (d *Database) onCompactionBegin(info pebble.CompactionInfo) {
 	if d.activeComp == 0 {
 		d.compStartTime = time.Now()

ethdb/remotedb/remotedb.go

@@ -22,6 +22,8 @@
 package remotedb
 
 import (
+	"errors"
+
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -32,6 +34,10 @@ type Database struct {
 	remote *rpc.Client
 }
 
+func (db *Database) CreateDBSnapshot(dir string) error {
+	return errors.New("createDBSnapshot method is not supported by remotedb")
+}
+
 func (db *Database) Has(key []byte) (bool, error) {
 	if _, err := db.Get(key); err != nil {
 		return false, nil

trie/trie_test.go

@@ -822,6 +822,9 @@ type spongeDb struct {
 	values  map[string]string
 }
 
+func (s *spongeDb) CreateDBSnapshot(dir string) error {
+	return errors.New("createDBSnapshot method is not supported by spongeDb")
+}
 func (s *spongeDb) Has(key []byte) (bool, error)             { panic("implement me") }
 func (s *spongeDb) Get(key []byte) ([]byte, error)           { return nil, errors.New("no such elem") }
 func (s *spongeDb) Delete(key []byte) error                  { panic("implement me") }