aristo: fold Adm column family into Vtx

execution_chain/db/aristo/aristo_blobify.nim

@@ -224,7 +224,7 @@ proc blobify*(vtx: VertexRef, key: HashKey): seq[byte] =
 
 proc blobifyTo*(lSst: SavedState; data: var seq[byte]) =
   ## Serialise a last saved state record
-  data.add lSst.key.data
+  data.add lSst.vTop.uint64.toBytesBE
   data.add lSst.serial.toBytesBE
   data.add @[0x7fu8]
 
@@ -340,19 +340,35 @@ proc deblobify*(record: openArray[byte], T: type HashKey): Opt[HashKey] =
 
 proc deblobify*(
     data: openArray[byte];
-    T: type SavedState;
-      ): Result[SavedState,AristoError] =
+    T: type SavedStateV0;
+      ): Result[SavedStateV0,AristoError] =
   ## De-serialise the last saved state data record previously encoded with
   ## `blobify()`.
   if data.len != 41:
     return err(DeblobWrongSize)
   if data[^1] != 0x7f:
     return err(DeblobWrongType)
 
-  ok(SavedState(
+  ok(SavedStateV0(
     key: Hash32(array[32, byte].initCopyFrom(data.toOpenArray(0, 31))),
     serial: uint64.fromBytesBE data.toOpenArray(32, 39)))
 
+proc deblobify*(
+    data: openArray[byte];
+    T: type SavedState;
+      ): Result[SavedState,AristoError] =
+  ## De-serialise the last saved state data record previously encoded with
+  ## `blobify()`.
+  if data.len != 17:
+    debugEcho "data ", data.len
+    return err(DeblobWrongSize)
+  if data[^1] != 0x7f:
+    return err(DeblobWrongType)
+
+  ok(SavedState(
+    vTop: VertexID(uint64.fromBytesBE data.toOpenArray(0, 7)),
+    serial: uint64.fromBytesBE data.toOpenArray(8, 15)))
+
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

execution_chain/db/aristo/aristo_desc.nim

@@ -97,12 +97,10 @@ type
     ## Backend interface.
     getVtxFn*: GetVtxFn              ## Read vertex record
     getKeyFn*: GetKeyFn              ## Read Merkle hash/key
-    getTuvFn*: GetTuvFn              ## Read top used vertex ID
     getLstFn*: GetLstFn              ## Read saved state
 
     putBegFn*: PutBegFn              ## Start bulk store session
     putVtxFn*: PutVtxFn              ## Bulk store vertex records
-    putTuvFn*: PutTuvFn              ## Store top used vertex ID
     putLstFn*: PutLstFn              ## Store saved state
     putEndFn*: PutEndFn              ## Commit bulk store session
 

execution_chain/db/aristo/aristo_desc/desc_backend.nim

@@ -57,12 +57,6 @@ type
         ## Generic backend database bulk storage function, `VertexRef(nil)`
         ## values indicate that records should be deleted.
 
-  PutTuvFn* =
-    proc(hdl: PutHdlRef; vs: VertexID)
-      {.gcsafe, raises: [].}
-        ## Generic backend database ID generator storage function for the
-        ## top used vertex ID.
-
   PutLstFn* =
     proc(hdl: PutHdlRef; lst: SavedState)
       {.gcsafe, raises: [].}

execution_chain/db/aristo/aristo_desc/desc_structural.nim

@@ -79,9 +79,15 @@ type
     vid*: VertexID                   ## Table lookup vertex ID (if any)
     vtx*: VertexRef                  ## Reference to vertex
 
+  SavedStateV0* = object
+    ## Legacy saved state from when state lived in separate column family
+    ## TODO remove before beta
+    key*: Hash32                     ## Some state hash (if any)
+    serial*: uint64                  ## Generic identifier from application
+
   SavedState* = object
     ## Last saved state
-    key*: Hash32                     ## Some state hash (if any)
+    vTop*: VertexID                   ## Top used VertexID
     serial*: uint64                  ## Generic identifier from application
 
   GetVtxFlag* = enum

execution_chain/db/aristo/aristo_get.nim

@@ -21,18 +21,12 @@ import
 # Public functions
 # ------------------------------------------------------------------------------
 
-proc getTuvBe*(
-    db: AristoDbRef;
-      ): Result[VertexID,AristoError] =
-  ## Get the ID generator state from the backened if available.
-  db.getTuvFn()
-
 proc getLstBe*(
     db: AristoDbRef;
       ): Result[SavedState,AristoError] =
   ## Get the last saved state
   db.getLstFn()
-  
+
 proc getVtxBe*(
     db: AristoDbRef;
     rvid: RootedVertexID;

execution_chain/db/aristo/aristo_init/init_common.nim

@@ -82,7 +82,7 @@ proc finishSession*(hdl: TypedPutHdlRef; db: TypedBackendRef) =
     db.txId = 0
 
 proc initInstance*(db: AristoDbRef): Result[void, AristoError] =
-  let vTop = ?db.getTuvFn()
+  let vTop = (?db.getLstFn()).vTop
   db.txRef = AristoTxRef(db: db, vTop: vTop, snapshot: Snapshot(level: Opt.some(0)))
   db.accLeaves = LruCache[Hash32, AccLeafRef].init(ACC_LRU_SIZE)
   db.stoLeaves = LruCache[Hash32, StoLeafRef].init(ACC_LRU_SIZE)

execution_chain/db/aristo/aristo_init/memory_db.nim

@@ -41,12 +41,10 @@ const
 type
   MemBackendRef* = ref object of TypedBackendRef
     sTab*: Table[RootedVertexID,seq[byte]] ## Structural vertex table making up a trie
-    tUvi*: Opt[VertexID]                   ## Top used vertex ID
     lSst*: Opt[SavedState]                 ## Last saved state
 
   MemPutHdlRef = ref object of TypedPutHdlRef
     sTab: Table[RootedVertexID,seq[byte]]
-    tUvi: Opt[VertexID]
     lSst: Opt[SavedState]
 
 when extraTraceMessages:
@@ -100,15 +98,10 @@ func getKeyFn(db: MemBackendRef): GetKeyFn =
         return ok((key, nil))
       err(GetKeyNotFound)
 
-func getTuvFn(db: MemBackendRef): GetTuvFn =
-  result =
-    proc(): Result[VertexID,AristoError]=
-      db.tUvi or ok(VertexID(0))
-
 func getLstFn(db: MemBackendRef): GetLstFn =
   result =
     proc(): Result[SavedState,AristoError]=
-      db.lSst or err(GetLstNotFound)
+      db.lSst or ok(default(SavedState))
 
 # -------------
 
@@ -128,13 +121,6 @@ func putVtxFn(db: MemBackendRef): PutVtxFn =
         else:
           hdl.sTab[rvid] = EmptyBlob
 
-func putTuvFn(db: MemBackendRef): PutTuvFn =
-  result =
-    proc(hdl: PutHdlRef; vs: VertexID)  =
-      let hdl = hdl.getSession db
-      if hdl.error.isNil:
-        hdl.tUvi = Opt.some(vs)
-
 func putLstFn(db: MemBackendRef): PutLstFn =
   result =
     proc(hdl: PutHdlRef; lst: SavedState) =
@@ -163,10 +149,6 @@ func putEndFn(db: MemBackendRef): PutEndFn =
         else:
           db.sTab.del vid
 
-      let tuv = hdl.tUvi.get(otherwise = VertexID(0))
-      if tuv.isValid:
-        db.tUvi = Opt.some(tuv)
-
       if hdl.lSst.isSome:
         db.lSst = hdl.lSst
 
@@ -190,12 +172,10 @@ func memoryBackend*(): AristoDbRef =
 
   db.getVtxFn = getVtxFn be
   db.getKeyFn = getKeyFn be
-  db.getTuvFn = getTuvFn be
   db.getLstFn = getLstFn be
 
   db.putBegFn = putBegFn be
   db.putVtxFn = putVtxFn be
-  db.putTuvFn = putTuvFn be
   db.putLstFn = putLstFn be
   db.putEndFn = putEndFn be
 

execution_chain/db/aristo/aristo_init/rocks_db.nim

@@ -104,33 +104,50 @@ proc getKeyFn(db: RdbBackendRef): GetKeyFn =
 
       err(GetKeyNotFound)
 
-proc getTuvFn(db: RdbBackendRef): GetTuvFn =
-  result =
-    proc(): Result[VertexID,AristoError]=
+proc getTuv(db: RdbBackendRef): Result[VertexID,AristoError]=
+  # Fetch serialised data record.
+  let data = db.rdb.getAdm(AdmTabIdTuv).valueOr:
+    when extraTraceMessages:
+      trace logTxt "getTuvFn: failed", error=error[0], info=error[1]
+    return err(error[0])
 
-      # Fetch serialised data record.
-      let data = db.rdb.getAdm(AdmTabIdTuv).valueOr:
-        when extraTraceMessages:
-          trace logTxt "getTuvFn: failed", error=error[0], info=error[1]
-        return err(error[0])
+  # Decode data record
+  if data.len == 0:
+    return ok VertexID(0)
 
-      # Decode data record
-      if data.len == 0:
-        return ok VertexID(0)
+  # Decode data record
+  data.deblobify VertexID
 
-      # Decode data record
-      result = data.deblobify VertexID
+proc getLstV0(db: RdbBackendRef): Result[SavedStateV0,AristoError] =
+  let data = db.rdb.getAdm(AdmTabIdLst).valueOr:
+    when extraTraceMessages:
+      trace logTxt "getTuvFn: failed", error=error[0], info=error[1]
+    return err(error[0])
+
+  if data.len == 0:
+    return ok default(SavedStateV0)
+
+  # Decode data record
+  data.deblobify SavedStateV0
 
 proc getLstFn(db: RdbBackendRef): GetLstFn =
   result =
     proc(): Result[SavedState,AristoError]=
 
       # Fetch serialised data record.
-      let data = db.rdb.getAdm(AdmTabIdLst).valueOr:
+      let data = db.rdb.getAdm().valueOr:
         when extraTraceMessages:
           trace logTxt "getLstFn: failed", error=error[0], info=error[1]
         return err(error[0])
 
+      if data.len == 0:
+        # TODO legacy database support, remove before beta
+        let
+          lst = ?db.getLstV0()
+          vTop = ?db.getTuv()
+
+        return ok(SavedState(vTop: vTop, serial: lst.serial))
+
       # Decode data record
       data.deblobify SavedState
 
@@ -153,33 +170,18 @@ proc putVtxFn(db: RdbBackendRef): PutVtxFn =
             code: error[1],
             info: error[2])
 
-proc putTuvFn(db: RdbBackendRef): PutTuvFn =
-  result =
-    proc(hdl: PutHdlRef; vs: VertexID)  =
-      let hdl = hdl.getSession db
-      if hdl.error.isNil:
-        if vs.isValid:
-          db.rdb.putAdm(hdl.session, AdmTabIdTuv, vs.blobify.data()).isOkOr:
-            hdl.error = TypedPutHdlErrRef(
-              pfx:  AdmPfx,
-              aid:  AdmTabIdTuv,
-              code: error[1],
-              info: error[2])
-            return
-
-
 proc putLstFn(db: RdbBackendRef): PutLstFn =
   result =
     proc(hdl: PutHdlRef; lst: SavedState) =
       let hdl = hdl.getSession db
       if hdl.error.isNil:
         let data = lst.blobify
-        db.rdb.putAdm(hdl.session, AdmTabIdLst, data).isOkOr:
+        db.rdb.putAdm(hdl.session, data).isOkOr:
           hdl.error = TypedPutHdlErrRef(
             pfx:  AdmPfx,
             aid:  AdmTabIdLst,
-            code: error[1],
-            info: error[2])
+            code: error[0],
+            info: error[1])
 
 proc putEndFn(db: RdbBackendRef): PutEndFn =
   result =
@@ -226,12 +228,10 @@ proc rocksDbBackend*(
 
   db.getVtxFn = getVtxFn be
   db.getKeyFn = getKeyFn be
-  db.getTuvFn = getTuvFn be
   db.getLstFn = getLstFn be
 
   db.putBegFn = putBegFn be
   db.putVtxFn = putVtxFn be
-  db.putTuvFn = putTuvFn be
   db.putLstFn = putLstFn be
   db.putEndFn = putEndFn be
 

execution_chain/db/aristo/aristo_init/rocks_db/rdb_desc.nim

@@ -24,6 +24,8 @@ import
 
 export minilru, rocksdb_desc
 
+const AdmKey* = default(seq[byte])
+
 type
   RdbWriteEventCb* =
     proc(session: WriteBatchRef): bool {.gcsafe, raises: [].}
@@ -38,7 +40,7 @@ type
 
   RdbInst* = object
     baseDb*: RocksDbInstanceRef
-    admCol*: ColFamilyReadWrite        ## Admin column family handler
+    admCol*: ColFamilyReadWrite        ## Legacy column family for administrative data
     vtxCol*: ColFamilyReadWrite        ## Vertex column family handler
 
     # Note that the key type `VertexID` for LRU caches requires that there is
@@ -64,7 +66,7 @@ type
 
   AristoCFs* = enum
     ## Column family symbols/handles and names used on the database
-    AdmCF = "AriAdm"                   ## Admin column family name
+    AdmCF = "AriAdm"                   ## Admin column family name (deprecated)
     VtxCF = "AriVtx"                   ## Vertex column family name
 
   RdbLruCounter* = array[bool, Atomic[uint64]]

execution_chain/db/aristo/aristo_init/rocks_db/rdb_get.nim

@@ -103,7 +103,11 @@ when defined(metrics):
 # Public functions
 # ------------------------------------------------------------------------------
 
+
 proc getAdm*(rdb: RdbInst, xid: AdminTabID): Result[seq[byte], (AristoError, string)] =
+  if isNil(rdb.admCol.handle()):
+    return ok(default(seq[byte]))
+
   var res: seq[byte]
   let onData = proc(data: openArray[byte]) =
     res = @data
@@ -119,6 +123,22 @@ proc getAdm*(rdb: RdbInst, xid: AdminTabID): Result[seq[byte], (AristoError, str
     res = EmptyBlob
   ok move(res)
 
+proc getAdm*(rdb: RdbInst): Result[seq[byte], (AristoError, string)] =
+  var res: seq[byte]
+  let onData = proc(data: openArray[byte]) =
+    res = @data
+
+  let gotData = rdb.vtxCol.get(AdmKey, onData).valueOr:
+    const errSym = RdbBeDriverGetAdmError
+    when extraTraceMessages:
+      trace logTxt "getAdm", xid, error = errSym, info = error
+    return err((errSym, error))
+
+  # Correct result if needed
+  if not gotData:
+    res = EmptyBlob
+  ok move(res)
+
 proc getKey*(
     rdb: var RdbInst, rvid: RootedVertexID, flags: set[GetVtxFlag]
 ): Result[(HashKey, VertexRef), (AristoError, string)] =

execution_chain/db/aristo/aristo_init/rocks_db/rdb_init.nim

@@ -114,8 +114,8 @@ proc init*(rdb: var RdbInst, opts: DbOptions, baseDb: RocksDbInstanceRef) =
     )
 
   # Initialise column handlers (this stores implicitely `baseDb`)
-  rdb.admCol = baseDb.db.getColFamily($AdmCF).valueOr:
-    raiseAssert "Cannot initialise AdmCF descriptor: " & error
+  rdb.admCol = baseDb.db.getColFamily($AdmCF).valueOr(default(ColFamilyReadWrite))
+
   rdb.vtxCol = baseDb.db.getColFamily($VtxCF).valueOr:
     raiseAssert "Cannot initialise VtxCF descriptor: " & error