[HF] MPT-20932 report migration progress

rest_api/rest_api_server/alembic/versions/c1d4f7a92b08_migrate_cloudaccount_config_to_fernet.py

@@ -32,6 +32,7 @@
 FERNET_PREFIX = 'v2:'
 FERNET_KDF_INFO = b'optscale-config-fernet-v1'
 BATCH_SIZE = 500
+PROGRESS_LOG_EVERY = 100
 
 
 def _etcd_salt():
@@ -81,54 +82,87 @@ def _iter_rows(conn):
         last_id = rows[-1][0]
 
 
+def _count_rows(conn):
+    stmt = text(
+        "SELECT COUNT(*) FROM cloudaccount "
+        "WHERE config IS NOT NULL AND deleted_at = 0"
+    )
+    return conn.execute(stmt).scalar()
+
+
+def _log_progress(operation, processed, total):
+    if processed % PROGRESS_LOG_EVERY == 0 or processed == total:
+        LOG.info(
+            '%s progress: processed=%d total=%d',
+            operation,
+            processed,
+            total,
+        )
+
+
 def upgrade():
     salt = _etcd_salt()
     fernet = _build_fernet(salt)
-    LOG.warning(
-        'migration salt: type=%s len=%s repr=%r',
-        type(salt).__name__, len(salt), salt[:8] if salt else salt)
 
     conn = op.get_bind()
+    total = _count_rows(conn)
+    LOG.info('cloudaccount config upgrade: total=%d', total)
     update_stmt = text(
         "UPDATE cloudaccount SET config = :cfg WHERE id = :id"
     )
-    migrated = skipped = failed = 0
+    migrated = skipped = failed = processed = 0
     for rows in _iter_rows(conn):
         for row_id, cfg in rows:
+            processed += 1
             if not cfg or cfg.startswith(FERNET_PREFIX):
                 skipped += 1
+                _log_progress('cloudaccount config upgrade', processed, total)
                 continue
             try:
                 plain = _legacy_decode(salt, cfg)
             except Exception:
                 failed += 1
+                _log_progress('cloudaccount config upgrade', processed, total)
                 continue
             new_cfg = _fernet_encode(fernet, plain)
             conn.execute(update_stmt, cfg=new_cfg, id=row_id)
             migrated += 1
+            _log_progress('cloudaccount config upgrade', processed, total)
     LOG.info(
-        'cloudaccount config migration: migrated=%d skipped=%d failed=%d',
-        migrated, skipped, failed)
+        'cloudaccount config migration: total=%d processed=%d migrated=%d skipped=%d failed=%d',
+        total, processed, migrated, skipped, failed)
 
 
 def downgrade():
     salt = _etcd_salt()
     fernet = _build_fernet(salt)
 
     conn = op.get_bind()
+    total = _count_rows(conn)
+    LOG.info('cloudaccount config downgrade: total=%d', total)
     update_stmt = text(
         "UPDATE cloudaccount SET config = :cfg WHERE id = :id"
     )
+    processed = restored = failed = 0
     for rows in _iter_rows(conn):
         for row_id, cfg in rows:
+            processed += 1
             if not cfg or not cfg.startswith(FERNET_PREFIX):
+                _log_progress('cloudaccount config downgrade', processed, total)
                 continue
             try:
                 token = cfg[len(FERNET_PREFIX):].encode('utf-8')
                 plain = json.loads(fernet.decrypt(token).decode('utf-8'))
             except InvalidToken:
+                failed += 1
                 LOG.exception(
                     'Failed to decode v2 config for %s', row_id)
+                _log_progress('cloudaccount config downgrade', processed, total)
                 continue
             legacy = cryptocode.encrypt(json.dumps(plain), salt)
             conn.execute(update_stmt, cfg=legacy, id=row_id)
+            restored += 1
+            _log_progress('cloudaccount config downgrade', processed, total)
+    LOG.info(
+        'cloudaccount config downgrade: total=%d processed=%d restored=%d failed=%d',
+        total, processed, restored, failed)