Added snapshot_name for versions, THIS MAKES THE EXPORT/IMPORT FORMAT INCOMPATIBLE

Author: wamdam

TEST

@@ -1,3 +1,5 @@
 #!/bin/bash
 
-alembic -c src/backy2/meta_backends/sql_migrations/alembic_cli.ini "$@"
+# ./alembic revision --autogenerate -m "Added snapshot_name to versions"
+
+alembic -c src/backy2/meta_backends/sql_migrations/alembic.ini "$@"

alembic

@@ -116,6 +116,7 @@ def __exit__(self, type, value, traceback):
         backy = backy_from_config(Config)()
         version_uid = backy.backup(
             'data-backup',
+            'snapshot-name',
             'file://'+os.path.join(testpath, 'data'),
             hints_from_rbd_diff(open(os.path.join(testpath, 'hints')).read()),
             from_version

smoketest.py

@@ -58,7 +58,7 @@ def __init__(self, meta_backend, data_backend, config, block_size=None,
         self.locking.unlock('backy')
 
 
-    def _prepare_version(self, name, size_bytes, from_version_uid=None):
+    def _prepare_version(self, name, snapshot_name, size_bytes, from_version_uid=None):
         """ Prepares the metadata for a new version.
         If from_version_uid is given, this is taken as the base, otherwise
         a pure sparse version is created.
@@ -72,7 +72,7 @@ def _prepare_version(self, name, size_bytes, from_version_uid=None):
             old_blocks = None
         size = math.ceil(size_bytes / self.block_size)
         # we always start with invalid versions, then validate them after backup
-        version_uid = self.meta_backend.set_version(name, size, size_bytes, 0)
+        version_uid = self.meta_backend.set_version(name, snapshot_name, size, size_bytes, 0)
         if not self.locking.lock(version_uid):
             raise LockError('Version {} is locked.'.format(version_uid))
         for id in range(size):
@@ -346,7 +346,7 @@ def rm(self, version_uid, force=True, disallow_rm_when_younger_than_days=0):
         self.locking.unlock(version_uid)
 
 
-    def backup(self, name, source, hints, from_version):
+    def backup(self, name, snapshot_name, source, hints, from_version):
         """ Create a backup from source.
         If hints are given, they must be tuples of (offset, length, exists)
         where offset and length are integers and exists is a boolean. Then, only
@@ -391,7 +391,7 @@ def backup(self, name, source, hints, from_version):
         read_blocks = set(read_blocks)
 
         try:
-            version_uid = self._prepare_version(name, source_size, from_version)
+            version_uid = self._prepare_version(name, snapshot_name, source_size, from_version)
         except RuntimeError as e:
             logger.error(str(e))
             logger.error('Backy exiting.')

src/backy2/backy.py

@@ -98,7 +98,7 @@ def read(self, version_uid, offset, length):
 
     def get_cow_version(self, from_version):
         cow_version_uid = self.backy._prepare_version(
-            'cow from {}'.format(from_version.uid),
+            'copy on write', from_version.uid,
             from_version.size_bytes, from_version.uid)
         self.cow[cow_version_uid] = {}  # contains version_uid: dict() of block id -> uid
         return cow_version_uid

src/backy2/enterprise/nbd.py

@@ -19,6 +19,9 @@ def init_logging(logfile, console_level):  # pragma: no cover
 
     logging.basicConfig(handlers = [console, logfile], level=logging.DEBUG)
 
+    # make alembic quiet
+    logging.getLogger('alembic').setLevel(logging.WARN)
+
     logger.info('$ ' + ' '.join(sys.argv))
 
 

src/backy2/logging.py

@@ -7,8 +7,8 @@ class MetaBackend():
     def __init__(self):
         pass
 
-    def set_version(self, version_name, size, size_bytes):
-        """ Creates a new version with a given name.
+    def set_version(self, version_name, snapshot_name, size, size_bytes):
+        """ Creates a new version with a given name and snapshot_name.
         size is the number of blocks this version will contain.
         Returns a uid for this version.
         """

src/backy2/meta_backends/__init__.py

@@ -16,7 +16,7 @@
 import uuid
 
 
-METADATA_VERSION = '2.1'
+METADATA_VERSION = '2.2'
 
 DELETE_CANDIDATE_MAYBE = 0
 DELETE_CANDIDATE_SURE = 1
@@ -47,15 +47,16 @@ class Version(Base):
     __tablename__ = 'versions'
     uid = Column(String(36), primary_key=True)
     date = Column("date", DateTime , default=func.now(), nullable=False)
-    name = Column(String, nullable=False)
+    name = Column(String, nullable=False, default='')
+    snapshot_name = Column(String, nullable=False, server_default='', default='')
     size = Column(BigInteger, nullable=False)
     size_bytes = Column(BigInteger, nullable=False)
     valid = Column(Integer, nullable=False)
     protected = Column(Integer, nullable=False)
 
     def __repr__(self):
-       return "<Version(uid='%s', name='%s', date='%s')>" % (
-                            self.uid, self.name, self.date)
+       return "<Version(uid='%s', name='%s', snapshot_name='%s', date='%s')>" % (
+                            self.uid, self.name, self.snapshot_name, self.date)
 
 
 DereferencedBlock = namedtuple('Block', ['uid', 'version_uid', 'id', 'date', 'checksum', 'size', 'valid'])
@@ -121,18 +122,20 @@ def __init__(self, config):
         from alembic.config import Config
         from alembic import command
         alembic_cfg = Config(os.path.join(os.path.dirname(os.path.realpath(__file__)), "sql_migrations", "alembic.ini"))
-        try:
-            Base.metadata.create_all(engine, checkfirst=False)  # checkfirst False will raise when it finds an existing table
-        except sqlalchemy.exc.OperationalError:
-            # tables already exist, see if there are any db schema upgrades
-            with engine.begin() as connection:
-                alembic_cfg.attributes['connection'] = connection
-                #command.upgrade(alembic_cfg, "head", sql=True)
-                command.upgrade(alembic_cfg, "head")
-        else:
-            # new database.
-            # mark the version table, "stamping" it with the most recent rev:
-            command.stamp(alembic_cfg, "head")
+        with engine.begin() as connection:
+            alembic_cfg.attributes['connection'] = connection
+            try:
+                Base.metadata.create_all(engine, checkfirst=False)  # checkfirst False will raise when it finds an existing table
+            except sqlalchemy.exc.OperationalError:
+                # tables already exist, see if there are any db schema upgrades
+                with engine.begin() as connection:
+                    alembic_cfg.attributes['connection'] = connection
+                    #command.upgrade(alembic_cfg, "head", sql=True)
+                    command.upgrade(alembic_cfg, "head")
+            else:
+                # new database.
+                # mark the version table, "stamping" it with the most recent rev:
+                command.stamp(alembic_cfg, "head")
 
         Session = sessionmaker(bind=engine)
         self.session = Session()
@@ -147,11 +150,12 @@ def _commit(self):
         self.session.commit()
 
 
-    def set_version(self, version_name, size, size_bytes, valid, protected=0):
+    def set_version(self, version_name, snapshot_name, size, size_bytes, valid, protected=0):
         uid = self._uid()
         version = Version(
             uid=uid,
             name=version_name,
+            snapshot_name=snapshot_name,
             size=size,
             size_bytes=size_bytes,
             valid=valid,
@@ -404,9 +408,11 @@ def export(self, version_uid, f):
             version.uid,
             version.date.strftime('%Y-%m-%d %H:%M:%S'),
             version.name,
+            version.snapshot_name,
             version.size,
             version.size_bytes,
             version.valid,
+            version.protected,
             ])
         for block in blocks:
             _csv.writerow([
@@ -426,7 +432,7 @@ def import_(self, f):
         signature = next(_csv)
         if signature[0] != 'backy2 Version {} metadata dump'.format(METADATA_VERSION):
             raise ValueError('Wrong import format.')
-        version_uid, version_date, version_name, version_size, version_size_bytes, version_valid = next(_csv)
+        version_uid, version_date, version_name, version_snapshot_name, version_size, version_size_bytes, version_valid, version_protected = next(_csv)
         try:
             self.get_version(version_uid)
         except KeyError:
@@ -437,9 +443,11 @@ def import_(self, f):
             uid=version_uid,
             date=datetime.datetime.strptime(version_date, '%Y-%m-%d %H:%M:%S'),
             name=version_name,
+            snapshot_name=version_snapshot_name,
             size=version_size,
             size_bytes=version_size_bytes,
             valid=version_valid,
+            protected=version_protected,
             )
         self.session.add(version)
         for uid, version_uid, id, date, checksum, size, valid in _csv:

src/backy2/meta_backends/sql.py

@@ -44,7 +44,7 @@ keys = generic
 
 [logger_root]
 level = WARN
-handlers = null
+handlers = console
 qualname =
 
 [logger_sqlalchemy]
@@ -54,7 +54,7 @@ qualname = sqlalchemy.engine
 
 [logger_alembic]
 level = INFO
-handlers = null
+handlers =
 qualname = alembic
 
 [handler_null]

src/backy2/meta_backends/sql_migrations/alembic.ini

@@ -13,13 +13,14 @@
 
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
-fileConfig(config.config_file_name)
+#fileConfig(config.config_file_name)
 
 # add your model's MetaData object here
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-target_metadata = None
+from backy2.meta_backends.sql import Base
+target_metadata = Base.metadata
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:

src/backy2/meta_backends/sql_migrations/alembic/env.py

@@ -0,0 +1,28 @@
+"""Added snapshot_name to versions
+
+Revision ID: bbe7904bfc3a
+Revises: 
+Create Date: 2017-04-11 18:11:04.786199
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'bbe7904bfc3a'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('versions', sa.Column('snapshot_name', sa.String(), server_default='', nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('versions', 'snapshot_name')
+    # ### end Alembic commands ###

src/backy2/meta_backends/sql_migrations/alembic/versions/bbe7904bfc3a_added_snapshot_name_to_versions.py

@@ -27,13 +27,13 @@ def __init__(self, machine_output, Config):
         self.backy = backy_from_config(Config)
 
 
-    def backup(self, name, source, rbd, from_version):
+    def backup(self, name, snapshot_name, source, rbd, from_version):
         backy = self.backy()
         hints = None
         if rbd:
             data = ''.join([line for line in fileinput.input(rbd).readline()])
             hints = hints_from_rbd_diff(data)
-        backy.backup(name, source, hints, from_version)
+        backy.backup(name, snapshot_name, source, hints, from_version)
         backy.close()
 
 
@@ -106,7 +106,7 @@ def _ls_blocks_machine_output(self, blocks):
     def _ls_versions_tbl_output(self, versions):
         tbl = PrettyTable()
         # TODO: number of invalid blocks, used disk space, shared disk space
-        tbl.field_names = ['date', 'name', 'size', 'size_bytes', 'uid',
+        tbl.field_names = ['date', 'name', 'snapshot_name', 'size', 'size_bytes', 'uid',
                 'valid', 'protected']
         tbl.align['name'] = 'l'
         tbl.align['size'] = 'r'
@@ -115,6 +115,7 @@ def _ls_versions_tbl_output(self, versions):
             tbl.add_row([
                 version.date,
                 version.name,
+                version.snapshot_name,
                 version.size,
                 version.size_bytes,
                 version.uid,
@@ -125,13 +126,14 @@ def _ls_versions_tbl_output(self, versions):
 
 
     def _ls_versions_machine_output(self, versions):
-        field_names = ['type', 'date', 'name', 'size', 'size_bytes', 'uid', 'valid', 'protected']
+        field_names = ['type', 'date', 'name', 'snapshot_name', 'size', 'size_bytes', 'uid', 'valid', 'protected']
         print(' '.join(field_names))
         for version in versions:
             print(' '.join(map(str, [
                 'version',
                 version.date,
                 version.name,
+                version.snapshot_name,
                 version.size,
                 version.size_bytes,
                 version.uid,
@@ -353,7 +355,8 @@ def main():
         help='Source (url-like, e.g. file:///dev/sda or rbd://pool/imagename@snapshot)')
     p.add_argument(
         'name',
-        help='Backup name')
+        help='Backup name (e.g. the hostname)')
+    p.add_argument('-s', '--snapshot-name', default='', help='Snapshot name (e.g. the name of the rbd snapshot)')
     p.add_argument('-r', '--rbd', default=None, help='Hints as rbd json format')
     p.add_argument('-f', '--from-version', default=None, help='Use this version-uid as base')
     p.set_defaults(func='backup')