diff --git a/execution_chain/db/aristo/aristo_blobify.nim b/execution_chain/db/aristo/aristo_blobify.nim index 4fa8cdc04a..26745a503f 100644 --- a/execution_chain/db/aristo/aristo_blobify.nim +++ b/execution_chain/db/aristo/aristo_blobify.nim @@ -224,7 +224,7 @@ proc blobify*(vtx: VertexRef, key: HashKey): seq[byte] = proc blobifyTo*(lSst: SavedState; data: var seq[byte]) = ## Serialise a last saved state record - data.add lSst.key.data + data.add lSst.vTop.uint64.toBytesBE data.add lSst.serial.toBytesBE data.add @[0x7fu8] @@ -340,8 +340,8 @@ proc deblobify*(record: openArray[byte], T: type HashKey): Opt[HashKey] = proc deblobify*( data: openArray[byte]; - T: type SavedState; - ): Result[SavedState,AristoError] = + T: type SavedStateV0; + ): Result[SavedStateV0,AristoError] = ## De-serialise the last saved state data record previously encoded with ## `blobify()`. if data.len != 41: @@ -349,10 +349,26 @@ proc deblobify*( if data[^1] != 0x7f: return err(DeblobWrongType) - ok(SavedState( + ok(SavedStateV0( key: Hash32(array[32, byte].initCopyFrom(data.toOpenArray(0, 31))), serial: uint64.fromBytesBE data.toOpenArray(32, 39))) +proc deblobify*( + data: openArray[byte]; + T: type SavedState; + ): Result[SavedState,AristoError] = + ## De-serialise the last saved state data record previously encoded with + ## `blobify()`. + if data.len != 17: + debugEcho "data ", data.len + return err(DeblobWrongSize) + if data[^1] != 0x7f: + return err(DeblobWrongType) + + ok(SavedState( + vTop: VertexID(uint64.fromBytesBE data.toOpenArray(0, 7)), + serial: uint64.fromBytesBE data.toOpenArray(8, 15))) + # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/execution_chain/db/aristo/aristo_desc.nim b/execution_chain/db/aristo/aristo_desc.nim index 12a7508426..3ad8182c47 100644 --- a/execution_chain/db/aristo/aristo_desc.nim +++ b/execution_chain/db/aristo/aristo_desc.nim @@ -97,12 +97,10 @@ type ## Backend interface. getVtxFn*: GetVtxFn ## Read vertex record getKeyFn*: GetKeyFn ## Read Merkle hash/key - getTuvFn*: GetTuvFn ## Read top used vertex ID getLstFn*: GetLstFn ## Read saved state putBegFn*: PutBegFn ## Start bulk store session putVtxFn*: PutVtxFn ## Bulk store vertex records - putTuvFn*: PutTuvFn ## Store top used vertex ID putLstFn*: PutLstFn ## Store saved state putEndFn*: PutEndFn ## Commit bulk store session diff --git a/execution_chain/db/aristo/aristo_desc/desc_backend.nim b/execution_chain/db/aristo/aristo_desc/desc_backend.nim index cb3cf8245b..fe294fe41d 100644 --- a/execution_chain/db/aristo/aristo_desc/desc_backend.nim +++ b/execution_chain/db/aristo/aristo_desc/desc_backend.nim @@ -57,12 +57,6 @@ type ## Generic backend database bulk storage function, `VertexRef(nil)` ## values indicate that records should be deleted. - PutTuvFn* = - proc(hdl: PutHdlRef; vs: VertexID) - {.gcsafe, raises: [].} - ## Generic backend database ID generator storage function for the - ## top used vertex ID. - PutLstFn* = proc(hdl: PutHdlRef; lst: SavedState) {.gcsafe, raises: [].} diff --git a/execution_chain/db/aristo/aristo_desc/desc_structural.nim b/execution_chain/db/aristo/aristo_desc/desc_structural.nim index 4cd75bef35..85fa93c3b4 100644 --- a/execution_chain/db/aristo/aristo_desc/desc_structural.nim +++ b/execution_chain/db/aristo/aristo_desc/desc_structural.nim @@ -79,9 +79,15 @@ type vid*: VertexID ## Table lookup vertex ID (if any) vtx*: VertexRef ## Reference to vertex + SavedStateV0* = object + ## Legacy saved state from when state lived in separate column family + ## TODO remove before beta + key*: Hash32 ## Some state hash (if any) + serial*: uint64 ## Generic identifier from application + SavedState* = object ## Last saved state - key*: Hash32 ## Some state hash (if any) + vTop*: VertexID ## Top used VertexID serial*: uint64 ## Generic identifier from application GetVtxFlag* = enum diff --git a/execution_chain/db/aristo/aristo_get.nim b/execution_chain/db/aristo/aristo_get.nim index 7deb7af626..184c89b79b 100644 --- a/execution_chain/db/aristo/aristo_get.nim +++ b/execution_chain/db/aristo/aristo_get.nim @@ -21,18 +21,12 @@ import # Public functions # ------------------------------------------------------------------------------ -proc getTuvBe*( - db: AristoDbRef; - ): Result[VertexID,AristoError] = - ## Get the ID generator state from the backened if available. - db.getTuvFn() - proc getLstBe*( db: AristoDbRef; ): Result[SavedState,AristoError] = ## Get the last saved state db.getLstFn() - + proc getVtxBe*( db: AristoDbRef; rvid: RootedVertexID; diff --git a/execution_chain/db/aristo/aristo_init/init_common.nim b/execution_chain/db/aristo/aristo_init/init_common.nim index 83ca547a37..73b0628d90 100644 --- a/execution_chain/db/aristo/aristo_init/init_common.nim +++ b/execution_chain/db/aristo/aristo_init/init_common.nim @@ -82,7 +82,7 @@ proc finishSession*(hdl: TypedPutHdlRef; db: TypedBackendRef) = db.txId = 0 proc initInstance*(db: AristoDbRef): Result[void, AristoError] = - let vTop = ?db.getTuvFn() + let vTop = (?db.getLstFn()).vTop db.txRef = AristoTxRef(db: db, vTop: vTop, snapshot: Snapshot(level: Opt.some(0))) db.accLeaves = LruCache[Hash32, AccLeafRef].init(ACC_LRU_SIZE) db.stoLeaves = LruCache[Hash32, StoLeafRef].init(ACC_LRU_SIZE) diff --git a/execution_chain/db/aristo/aristo_init/memory_db.nim b/execution_chain/db/aristo/aristo_init/memory_db.nim index 1e0379a4ef..b0d5c35d49 100644 --- a/execution_chain/db/aristo/aristo_init/memory_db.nim +++ b/execution_chain/db/aristo/aristo_init/memory_db.nim @@ -41,12 +41,10 @@ const type MemBackendRef* = ref object of TypedBackendRef sTab*: Table[RootedVertexID,seq[byte]] ## Structural vertex table making up a trie - tUvi*: Opt[VertexID] ## Top used vertex ID lSst*: Opt[SavedState] ## Last saved state MemPutHdlRef = ref object of TypedPutHdlRef sTab: Table[RootedVertexID,seq[byte]] - tUvi: Opt[VertexID] lSst: Opt[SavedState] when extraTraceMessages: @@ -100,15 +98,10 @@ func getKeyFn(db: MemBackendRef): GetKeyFn = return ok((key, nil)) err(GetKeyNotFound) -func getTuvFn(db: MemBackendRef): GetTuvFn = - result = - proc(): Result[VertexID,AristoError]= - db.tUvi or ok(VertexID(0)) - func getLstFn(db: MemBackendRef): GetLstFn = result = proc(): Result[SavedState,AristoError]= - db.lSst or err(GetLstNotFound) + db.lSst or ok(default(SavedState)) # ------------- @@ -128,13 +121,6 @@ func putVtxFn(db: MemBackendRef): PutVtxFn = else: hdl.sTab[rvid] = EmptyBlob -func putTuvFn(db: MemBackendRef): PutTuvFn = - result = - proc(hdl: PutHdlRef; vs: VertexID) = - let hdl = hdl.getSession db - if hdl.error.isNil: - hdl.tUvi = Opt.some(vs) - func putLstFn(db: MemBackendRef): PutLstFn = result = proc(hdl: PutHdlRef; lst: SavedState) = @@ -163,10 +149,6 @@ func putEndFn(db: MemBackendRef): PutEndFn = else: db.sTab.del vid - let tuv = hdl.tUvi.get(otherwise = VertexID(0)) - if tuv.isValid: - db.tUvi = Opt.some(tuv) - if hdl.lSst.isSome: db.lSst = hdl.lSst @@ -190,12 +172,10 @@ func memoryBackend*(): AristoDbRef = db.getVtxFn = getVtxFn be db.getKeyFn = getKeyFn be - db.getTuvFn = getTuvFn be db.getLstFn = getLstFn be db.putBegFn = putBegFn be db.putVtxFn = putVtxFn be - db.putTuvFn = putTuvFn be db.putLstFn = putLstFn be db.putEndFn = putEndFn be diff --git a/execution_chain/db/aristo/aristo_init/rocks_db.nim b/execution_chain/db/aristo/aristo_init/rocks_db.nim index 240c6aa4e2..d5b1a98461 100644 --- a/execution_chain/db/aristo/aristo_init/rocks_db.nim +++ b/execution_chain/db/aristo/aristo_init/rocks_db.nim @@ -104,33 +104,50 @@ proc getKeyFn(db: RdbBackendRef): GetKeyFn = err(GetKeyNotFound) -proc getTuvFn(db: RdbBackendRef): GetTuvFn = - result = - proc(): Result[VertexID,AristoError]= +proc getTuv(db: RdbBackendRef): Result[VertexID,AristoError]= + # Fetch serialised data record. + let data = db.rdb.getAdm(AdmTabIdTuv).valueOr: + when extraTraceMessages: + trace logTxt "getTuvFn: failed", error=error[0], info=error[1] + return err(error[0]) - # Fetch serialised data record. - let data = db.rdb.getAdm(AdmTabIdTuv).valueOr: - when extraTraceMessages: - trace logTxt "getTuvFn: failed", error=error[0], info=error[1] - return err(error[0]) + # Decode data record + if data.len == 0: + return ok VertexID(0) - # Decode data record - if data.len == 0: - return ok VertexID(0) + # Decode data record + data.deblobify VertexID - # Decode data record - result = data.deblobify VertexID +proc getLstV0(db: RdbBackendRef): Result[SavedStateV0,AristoError] = + let data = db.rdb.getAdm(AdmTabIdLst).valueOr: + when extraTraceMessages: + trace logTxt "getTuvFn: failed", error=error[0], info=error[1] + return err(error[0]) + + if data.len == 0: + return ok default(SavedStateV0) + + # Decode data record + data.deblobify SavedStateV0 proc getLstFn(db: RdbBackendRef): GetLstFn = result = proc(): Result[SavedState,AristoError]= # Fetch serialised data record. - let data = db.rdb.getAdm(AdmTabIdLst).valueOr: + let data = db.rdb.getAdm().valueOr: when extraTraceMessages: trace logTxt "getLstFn: failed", error=error[0], info=error[1] return err(error[0]) + if data.len == 0: + # TODO legacy database support, remove before beta + let + lst = ?db.getLstV0() + vTop = ?db.getTuv() + + return ok(SavedState(vTop: vTop, serial: lst.serial)) + # Decode data record data.deblobify SavedState @@ -153,33 +170,18 @@ proc putVtxFn(db: RdbBackendRef): PutVtxFn = code: error[1], info: error[2]) -proc putTuvFn(db: RdbBackendRef): PutTuvFn = - result = - proc(hdl: PutHdlRef; vs: VertexID) = - let hdl = hdl.getSession db - if hdl.error.isNil: - if vs.isValid: - db.rdb.putAdm(hdl.session, AdmTabIdTuv, vs.blobify.data()).isOkOr: - hdl.error = TypedPutHdlErrRef( - pfx: AdmPfx, - aid: AdmTabIdTuv, - code: error[1], - info: error[2]) - return - - proc putLstFn(db: RdbBackendRef): PutLstFn = result = proc(hdl: PutHdlRef; lst: SavedState) = let hdl = hdl.getSession db if hdl.error.isNil: let data = lst.blobify - db.rdb.putAdm(hdl.session, AdmTabIdLst, data).isOkOr: + db.rdb.putAdm(hdl.session, data).isOkOr: hdl.error = TypedPutHdlErrRef( pfx: AdmPfx, aid: AdmTabIdLst, - code: error[1], - info: error[2]) + code: error[0], + info: error[1]) proc putEndFn(db: RdbBackendRef): PutEndFn = result = @@ -226,12 +228,10 @@ proc rocksDbBackend*( db.getVtxFn = getVtxFn be db.getKeyFn = getKeyFn be - db.getTuvFn = getTuvFn be db.getLstFn = getLstFn be db.putBegFn = putBegFn be db.putVtxFn = putVtxFn be - db.putTuvFn = putTuvFn be db.putLstFn = putLstFn be db.putEndFn = putEndFn be diff --git a/execution_chain/db/aristo/aristo_init/rocks_db/rdb_desc.nim b/execution_chain/db/aristo/aristo_init/rocks_db/rdb_desc.nim index 26b41983a4..fb047789f0 100644 --- a/execution_chain/db/aristo/aristo_init/rocks_db/rdb_desc.nim +++ b/execution_chain/db/aristo/aristo_init/rocks_db/rdb_desc.nim @@ -24,6 +24,8 @@ import export minilru, rocksdb_desc +const AdmKey* = default(seq[byte]) + type RdbWriteEventCb* = proc(session: WriteBatchRef): bool {.gcsafe, raises: [].} @@ -38,7 +40,7 @@ type RdbInst* = object baseDb*: RocksDbInstanceRef - admCol*: ColFamilyReadWrite ## Admin column family handler + admCol*: ColFamilyReadWrite ## Legacy column family for administrative data vtxCol*: ColFamilyReadWrite ## Vertex column family handler # Note that the key type `VertexID` for LRU caches requires that there is @@ -64,7 +66,7 @@ type AristoCFs* = enum ## Column family symbols/handles and names used on the database - AdmCF = "AriAdm" ## Admin column family name + AdmCF = "AriAdm" ## Admin column family name (deprecated) VtxCF = "AriVtx" ## Vertex column family name RdbLruCounter* = array[bool, Atomic[uint64]] diff --git a/execution_chain/db/aristo/aristo_init/rocks_db/rdb_get.nim b/execution_chain/db/aristo/aristo_init/rocks_db/rdb_get.nim index 15ec67e5a2..6f0669c343 100644 --- a/execution_chain/db/aristo/aristo_init/rocks_db/rdb_get.nim +++ b/execution_chain/db/aristo/aristo_init/rocks_db/rdb_get.nim @@ -103,7 +103,11 @@ when defined(metrics): # Public functions # ------------------------------------------------------------------------------ + proc getAdm*(rdb: RdbInst, xid: AdminTabID): Result[seq[byte], (AristoError, string)] = + if isNil(rdb.admCol.handle()): + return ok(default(seq[byte])) + var res: seq[byte] let onData = proc(data: openArray[byte]) = res = @data @@ -119,6 +123,22 @@ proc getAdm*(rdb: RdbInst, xid: AdminTabID): Result[seq[byte], (AristoError, str res = EmptyBlob ok move(res) +proc getAdm*(rdb: RdbInst): Result[seq[byte], (AristoError, string)] = + var res: seq[byte] + let onData = proc(data: openArray[byte]) = + res = @data + + let gotData = rdb.vtxCol.get(AdmKey, onData).valueOr: + const errSym = RdbBeDriverGetAdmError + when extraTraceMessages: + trace logTxt "getAdm", xid, error = errSym, info = error + return err((errSym, error)) + + # Correct result if needed + if not gotData: + res = EmptyBlob + ok move(res) + proc getKey*( rdb: var RdbInst, rvid: RootedVertexID, flags: set[GetVtxFlag] ): Result[(HashKey, VertexRef), (AristoError, string)] = diff --git a/execution_chain/db/aristo/aristo_init/rocks_db/rdb_init.nim b/execution_chain/db/aristo/aristo_init/rocks_db/rdb_init.nim index b875846d58..6670195d03 100644 --- a/execution_chain/db/aristo/aristo_init/rocks_db/rdb_init.nim +++ b/execution_chain/db/aristo/aristo_init/rocks_db/rdb_init.nim @@ -114,8 +114,8 @@ proc init*(rdb: var RdbInst, opts: DbOptions, baseDb: RocksDbInstanceRef) = ) # Initialise column handlers (this stores implicitely `baseDb`) - rdb.admCol = baseDb.db.getColFamily($AdmCF).valueOr: - raiseAssert "Cannot initialise AdmCF descriptor: " & error + rdb.admCol = baseDb.db.getColFamily($AdmCF).valueOr(default(ColFamilyReadWrite)) + rdb.vtxCol = baseDb.db.getColFamily($VtxCF).valueOr: raiseAssert "Cannot initialise VtxCF descriptor: " & error diff --git a/execution_chain/db/aristo/aristo_init/rocks_db/rdb_put.nim b/execution_chain/db/aristo/aristo_init/rocks_db/rdb_put.nim index 1a5ee71bd0..1187e4c1b5 100644 --- a/execution_chain/db/aristo/aristo_init/rocks_db/rdb_put.nim +++ b/execution_chain/db/aristo/aristo_init/rocks_db/rdb_put.nim @@ -17,7 +17,6 @@ import rocksdb, results, ../../[aristo_blobify, aristo_desc], - ../init_common, ./rdb_desc const @@ -56,22 +55,21 @@ proc commit*(rdb: var RdbInst, session: SharedWriteBatchRef): Result[void,(Arist proc putAdm*( rdb: var RdbInst; session: SharedWriteBatchRef, - xid: AdminTabID; data: openArray[byte]; - ): Result[void,(AdminTabID,AristoError,string)] = + ): Result[void,(AristoError,string)] = let dsc = session.batch if data.len == 0: - dsc.delete(xid.toOpenArray, rdb.admCol.handle()).isOkOr: + dsc.delete(AdmKey, rdb.vtxCol.handle()).isOkOr: const errSym = RdbBeDriverDelAdmError when extraTraceMessages: trace logTxt "putAdm()", xid, error=errSym, info=error - return err((xid,errSym,error)) + return err((errSym,error)) else: - dsc.put(xid.toOpenArray, data, rdb.admCol.handle()).isOkOr: + dsc.put(AdmKey, data, rdb.vtxCol.handle()).isOkOr: const errSym = RdbBeDriverPutAdmError when extraTraceMessages: trace logTxt "putAdm()", xid, error=errSym, info=error - return err((xid,errSym,error)) + return err((errSym,error)) ok() proc putVtx*( diff --git a/execution_chain/db/aristo/aristo_tx_frame.nim b/execution_chain/db/aristo/aristo_tx_frame.nim index a9f87fbe30..2b1cfb4882 100644 --- a/execution_chain/db/aristo/aristo_tx_frame.nim +++ b/execution_chain/db/aristo/aristo_tx_frame.nim @@ -118,7 +118,7 @@ proc persist*( return let lSst = SavedState( - key: emptyRoot, # placeholder for more + vTop: txFrame.vTop, serial: txFrame.blockNumber.expect("`checkpoint` before persisting frame"), ) @@ -203,7 +203,6 @@ with --debug-eager-state-root.""" do: db.putVtxFn(batch, rvid, vtx, default(HashKey)) - db.putTuvFn(batch, txFrame.vTop) db.putLstFn(batch, lSst) # TODO above, we only prepare the changes to the database but don't actually diff --git a/execution_chain/db/core_db/backend/aristo_rocksdb.nim b/execution_chain/db/core_db/backend/aristo_rocksdb.nim index 190f3b32de..ec8c0f3769 100644 --- a/execution_chain/db/core_db/backend/aristo_rocksdb.nim +++ b/execution_chain/db/core_db/backend/aristo_rocksdb.nim @@ -152,9 +152,12 @@ proc newRocksDbCoreDbRef*(basePath: string, opts: DbOptions): CoreDbRef = # The same column family options are used for all column families meaning that # the options are a compromise between the various write and access patterns # of what's stored in there - there's room for improvement here! + + # Legacy support: adm CF, if it exists + let (dbOpts, cfOpts) = opts.toRocksDb() - cfDescs = (AristoCFs.items().toSeq().mapIt($it) & KvtCFs.items().toSeq().mapIt($it)) + cfDescs = @[$AristoCFs.VtxCF] & KvtCFs.items().toSeq().mapIt($it) baseDb = RocksDbInstanceRef.open(basePath, dbOpts, cfOpts, cfDescs).expect( "Open database from " & basePath )