From 91061493c88589f557f5fa5d6f1f637b43d8772c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Fri, 4 Nov 2022 12:59:07 +0100
Subject: [PATCH 01/17] support in cliain for version_upgrade

---
 aleph-client/src/version_upgrade.rs   |  3 ++-
 bin/cliain/src/commands.rs            | 35 ++++++++++++++++++++++++---
 bin/cliain/src/lib.rs                 |  2 ++
 bin/cliain/src/main.rs                | 14 ++++++++---
 bin/cliain/src/version_upgrade.rs     | 20 +++++++++++++++
 e2e-tests/src/test/version_upgrade.rs | 16 +++++++++---
 6 files changed, 80 insertions(+), 10 deletions(-)
 create mode 100644 bin/cliain/src/version_upgrade.rs

diff --git a/aleph-client/src/version_upgrade.rs b/aleph-client/src/version_upgrade.rs
index 5265aaf6d3..bc3150f819 100644
--- a/aleph-client/src/version_upgrade.rs
+++ b/aleph-client/src/version_upgrade.rs
@@ -10,6 +10,7 @@ pub fn schedule_upgrade(
     connection: &RootConnection,
     version: Version,
     session: SessionIndex,
+    state: XtStatus,
 ) -> Result<(), ApiClientError> {
     let connection = connection.as_connection();
     let upgrade_call = compose_call!(
@@ -30,7 +31,7 @@ pub fn schedule_upgrade(
         &connection,
         xt,
         Some("schedule finality version change"),
-        XtStatus::Finalized,
+        state,
     )
     .map(|_| ())
 }
diff --git a/bin/cliain/src/commands.rs b/bin/cliain/src/commands.rs
index adeea04533..ab49ce095b 100644
--- a/bin/cliain/src/commands.rs
+++ b/bin/cliain/src/commands.rs
@@ -3,9 +3,9 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use aleph_client::BlockNumber;
-use clap::{Args, Subcommand};
-use primitives::{Balance, CommitteeSeats};
+use aleph_client::{BlockNumber, XtStatus};
+use clap::{clap_derive::ValueEnum, Args, Subcommand};
+use primitives::{Balance, CommitteeSeats, SessionIndex};
 use serde::{Deserialize, Serialize};
 use sp_core::H256;
 use substrate_api_client::AccountId;
@@ -111,6 +111,8 @@ pub struct ChangeValidatorArgs {
     pub committee_size: Option<CommitteeSeats>,
 }
 
+pub type Version = u32;
+
 impl std::str::FromStr for ChangeValidatorArgs {
     type Err = serde_json::Error;
 
@@ -124,6 +126,21 @@ impl std::str::FromStr for ChangeValidatorArgs {
     }
 }
 
+#[derive(Debug, Clone, ValueEnum)]
+pub enum ExtrinsicState {
+    InBlock,
+    Finalized,
+}
+
+impl From<ExtrinsicState> for XtStatus {
+    fn from(state: ExtrinsicState) -> Self {
+        match state {
+            ExtrinsicState::InBlock => XtStatus::InBlock,
+            ExtrinsicState::Finalized => XtStatus::Finalized,
+        }
+    }
+}
+
 #[derive(Debug, Clone, Subcommand)]
 pub enum Command {
     /// Staking call to bond stash with controller
@@ -336,4 +353,16 @@ pub enum Command {
     /// Code can only be removed by its original uploader (its owner) and only if it is not used by any contract.
     /// API signature: https://polkadot.js.org/docs/substrate/extrinsics/#removecodecode_hash-h256
     ContractRemoveCode(ContractRemoveCode),
+
+    /// Schedules a version upgrade of the network.
+    VersionUpgradeSchedule {
+        #[clap(long)]
+        version: Version,
+
+        #[clap(long)]
+        session: SessionIndex,
+
+        #[clap(long, value_enum, default_value_t=ExtrinsicState::Finalized)]
+        expected_state: ExtrinsicState,
+    },
 }
diff --git a/bin/cliain/src/lib.rs b/bin/cliain/src/lib.rs
index b69fea25fd..6a50a9b4ad 100644
--- a/bin/cliain/src/lib.rs
+++ b/bin/cliain/src/lib.rs
@@ -8,6 +8,7 @@ mod staking;
 mod transfer;
 mod treasury;
 mod validators;
+mod version_upgrade;
 mod vesting;
 
 use aleph_client::{
@@ -27,6 +28,7 @@ pub use treasury::{
     approve as treasury_approve, propose as treasury_propose, reject as treasury_reject,
 };
 pub use validators::change_validators;
+pub use version_upgrade::schedule_upgrade;
 pub use vesting::{vest, vest_other, vested_transfer};
 
 pub struct ConnectionConfig {
diff --git a/bin/cliain/src/main.rs b/bin/cliain/src/main.rs
index 74338c9384..0e2bea0f39 100644
--- a/bin/cliain/src/main.rs
+++ b/bin/cliain/src/main.rs
@@ -8,9 +8,9 @@ use clap::Parser;
 use cliain::{
     bond, call, change_validators, finalize, force_new_era, instantiate, instantiate_with_code,
     next_session_keys, nominate, owner_info, prepare_keys, prompt_password_hidden, remove_code,
-    rotate_keys, set_emergency_finalizer, set_keys, set_staking_limits, transfer, treasury_approve,
-    treasury_propose, treasury_reject, update_runtime, upload_code, validate, vest, vest_other,
-    vested_transfer, Command, ConnectionConfig,
+    rotate_keys, schedule_upgrade, set_emergency_finalizer, set_keys, set_staking_limits, transfer,
+    treasury_approve, treasury_propose, treasury_reject, update_runtime, upload_code, validate,
+    vest, vest_other, vested_transfer, Command, ConnectionConfig,
 };
 use log::{error, info};
 use sp_core::Pair;
@@ -182,6 +182,14 @@ fn main() {
             Ok(result) => println!("{:?}", result),
             Err(why) => error!("Contract remove code failed {:?}", why),
         },
+        Command::VersionUpgradeSchedule {
+            version,
+            session: session_for_upgrade,
+            expected_state,
+        } => match schedule_upgrade(cfg.into(), version, session_for_upgrade, expected_state) {
+            Ok(_) => {}
+            Err(why) => error!("Unable to schedule an upgrade {:?}", why),
+        },
     }
 }
 
diff --git a/bin/cliain/src/version_upgrade.rs b/bin/cliain/src/version_upgrade.rs
new file mode 100644
index 0000000000..a38cf8913e
--- /dev/null
+++ b/bin/cliain/src/version_upgrade.rs
@@ -0,0 +1,20 @@
+use aleph_client::RootConnection;
+use anyhow::Error;
+use primitives::SessionIndex;
+
+use crate::commands::{ExtrinsicState, Version};
+
+pub fn schedule_upgrade(
+    connection: RootConnection,
+    version: Version,
+    session_for_upgrade: SessionIndex,
+    expected_state: ExtrinsicState,
+) -> anyhow::Result<()> {
+    aleph_client::schedule_upgrade(
+        &connection,
+        version,
+        session_for_upgrade,
+        expected_state.into(),
+    )
+    .map_err(Error::new)
+}
diff --git a/e2e-tests/src/test/version_upgrade.rs b/e2e-tests/src/test/version_upgrade.rs
index 1dbd3a5193..b6231c4823 100644
--- a/e2e-tests/src/test/version_upgrade.rs
+++ b/e2e-tests/src/test/version_upgrade.rs
@@ -1,6 +1,6 @@
 use aleph_client::{
     get_current_session, get_session_period, schedule_upgrade, wait_for_at_least_session,
-    wait_for_finalized_block, AnyConnection,
+    wait_for_finalized_block, AnyConnection, XtStatus,
 };
 use primitives::{Header, SessionIndex};
 
@@ -28,7 +28,12 @@ pub fn schedule_version_change(config: &Config) -> anyhow::Result<()> {
         .unwrap_or(UPGRADE_FINALIZATION_WAIT_SESSIONS);
     let session_after_upgrade = session_for_upgrade + wait_sessions_after_upgrade;
 
-    schedule_upgrade(&connection, version_for_upgrade, session_for_upgrade)?;
+    schedule_upgrade(
+        &connection,
+        version_for_upgrade,
+        session_for_upgrade,
+        XtStatus::Finalized,
+    )?;
 
     wait_for_at_least_session(&connection, session_after_upgrade)?;
     let block_number = session_after_upgrade * get_session_period(&connection);
@@ -56,7 +61,12 @@ pub fn schedule_doomed_version_change_and_verify_finalization_stopped(
         .unwrap_or(UPGRADE_FINALIZATION_WAIT_SESSIONS);
     let session_after_upgrade = session_for_upgrade + wait_sessions_after_upgrade;
 
-    schedule_upgrade(&connection, version_for_upgrade, session_for_upgrade)?;
+    schedule_upgrade(
+        &connection,
+        version_for_upgrade,
+        session_for_upgrade,
+        XtStatus::Finalized,
+    )?;
     wait_for_at_least_session(&connection, session_for_upgrade)?;
     let last_finalized_block = session_for_upgrade * get_session_period(&connection) - 1;
 

From a13cc8f161bfb11dccabf6c25b4f4bf025aa907c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Fri, 4 Nov 2022 12:59:50 +0100
Subject: [PATCH 02/17] support of our docker-compose to run using bridged
 docker's network

---
 docker/common.yml                 |  1 -
 docker/docker-compose.base.yml    | 75 +++++++++++++++++++++++++++++++
 docker/docker-compose.bridged.yml | 62 +++++++++++++++++++++++++
 docker/docker-compose.yml         | 73 +++++++-----------------------
 4 files changed, 152 insertions(+), 59 deletions(-)
 create mode 100644 docker/docker-compose.base.yml
 create mode 100644 docker/docker-compose.bridged.yml

diff --git a/docker/common.yml b/docker/common.yml
index d41d7f41b2..59a04adf45 100644
--- a/docker/common.yml
+++ b/docker/common.yml
@@ -1,7 +1,6 @@
 services:
   AlephNodeService:
     image: aleph-node:latest
-    network_mode: host
     environment:
       - PURGE_BEFORE_START=true
       - RUST_LOG=info
diff --git a/docker/docker-compose.base.yml b/docker/docker-compose.base.yml
new file mode 100644
index 0000000000..c2957f7be7
--- /dev/null
+++ b/docker/docker-compose.base.yml
@@ -0,0 +1,75 @@
+# When increasing number of nodes in this file, change default value of --validators-count param in e2e-tests/src/config.rs
+
+services:
+  Node0:
+    extends:
+      file: common.yml
+      service: AlephBootNode
+    container_name: Node0
+    environment:
+      - RPC_PORT=9933
+      - WS_PORT=9943
+      - PORT=30333
+      - VALIDATOR_PORT=30343
+      - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30343
+      - NAME=Node0
+
+  Node1:
+    extends:
+      file: common.yml
+      service: AlephNonBootNode
+    container_name: Node1
+    environment:
+      - RPC_PORT=9934
+      - WS_PORT=9944
+      - PORT=30334
+      - VALIDATOR_PORT=30344
+      - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30344
+      - NAME=Node1
+      - BASE_PATH=/data/5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o
+      - NODE_KEY_PATH=/data/5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o/p2p_secret
+
+  Node2:
+    extends:
+      file: common.yml
+      service: AlephNonBootNode
+    container_name: Node2
+    environment:
+      - RPC_PORT=9935
+      - WS_PORT=9945
+      - PORT=30335
+      - VALIDATOR_PORT=30345
+      - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30345
+      - NAME=Node2
+      - BASE_PATH=/data/5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9
+      - NODE_KEY_PATH=/data/5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9/p2p_secret
+
+  Node3:
+    extends:
+      file: common.yml
+      service: AlephNonBootNode
+    container_name: Node3
+    environment:
+      - RPC_PORT=9936
+      - WS_PORT=9946
+      - PORT=30336
+      - VALIDATOR_PORT=30346
+      - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30346
+      - NAME=Node3
+      - BASE_PATH=/data/5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK
+      - NODE_KEY_PATH=/data/5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK/p2p_secret
+
+  Node4:
+    extends:
+      file: common.yml
+      service: AlephNonBootNode
+    container_name: Node4
+    environment:
+      - RPC_PORT=9937
+      - WS_PORT=9947
+      - PORT=30337
+      - VALIDATOR_PORT=30347
+      - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30347
+      - NAME=Node4
+      - BASE_PATH=/data/5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW
+      - NODE_KEY_PATH=/data/5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW/p2p_secret
diff --git a/docker/docker-compose.bridged.yml b/docker/docker-compose.bridged.yml
new file mode 100644
index 0000000000..2ee3ddc132
--- /dev/null
+++ b/docker/docker-compose.bridged.yml
@@ -0,0 +1,62 @@
+services:
+  Node0:
+    extends:
+      file: docker-compose.base.yml
+      service: Node0
+    networks:
+      - main
+      - Node0
+
+  Node1:
+    extends:
+      file: docker-compose.base.yml
+      service: Node1
+    networks:
+      - main
+      - Node1
+    environment:
+      - BOOT_NODES=/dns4/Node0/tcp/30333/p2p/$BOOTNODE_PEER_ID
+
+  Node2:
+    extends:
+      file: docker-compose.base.yml
+      service: Node2
+    networks:
+      - main
+      - Node2
+    environment:
+      - BOOT_NODES=/dns4/Node0/tcp/30333/p2p/$BOOTNODE_PEER_ID
+
+  Node3:
+    extends:
+      file: docker-compose.base.yml
+      service: Node3
+    networks:
+      - main
+      - Node3
+    environment:
+      - BOOT_NODES=/dns4/Node0/tcp/30333/p2p/$BOOTNODE_PEER_ID
+
+  Node4:
+    extends:
+      file: docker-compose.base.yml
+      service: Node4
+    networks:
+      - main
+      - Node4
+    environment:
+      - BOOT_NODES=/dns4/Node0/tcp/30333/p2p/$BOOTNODE_PEER_ID
+
+networks:
+  main:
+    name: main-network
+  Node0:
+    name: Node0-network
+  Node1:
+    name: Node1-network
+  Node2:
+    name: Node2-network
+  Node3:
+    name: Node3-network
+  Node4:
+    name: Node4-network
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index c2957f7be7..154031e825 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -3,73 +3,30 @@
 services:
   Node0:
     extends:
-      file: common.yml
-      service: AlephBootNode
-    container_name: Node0
-    environment:
-      - RPC_PORT=9933
-      - WS_PORT=9943
-      - PORT=30333
-      - VALIDATOR_PORT=30343
-      - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30343
-      - NAME=Node0
+      file: docker-compose.base.yml
+      service: Node0
+    network_mode: host
 
   Node1:
     extends:
-      file: common.yml
-      service: AlephNonBootNode
-    container_name: Node1
-    environment:
-      - RPC_PORT=9934
-      - WS_PORT=9944
-      - PORT=30334
-      - VALIDATOR_PORT=30344
-      - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30344
-      - NAME=Node1
-      - BASE_PATH=/data/5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o
-      - NODE_KEY_PATH=/data/5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o/p2p_secret
+      file: docker-compose.base.yml
+      service: Node1
+    network_mode: host
 
   Node2:
     extends:
-      file: common.yml
-      service: AlephNonBootNode
-    container_name: Node2
-    environment:
-      - RPC_PORT=9935
-      - WS_PORT=9945
-      - PORT=30335
-      - VALIDATOR_PORT=30345
-      - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30345
-      - NAME=Node2
-      - BASE_PATH=/data/5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9
-      - NODE_KEY_PATH=/data/5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9/p2p_secret
+      file: docker-compose.base.yml
+      service: Node2
+    network_mode: host
 
   Node3:
     extends:
-      file: common.yml
-      service: AlephNonBootNode
-    container_name: Node3
-    environment:
-      - RPC_PORT=9936
-      - WS_PORT=9946
-      - PORT=30336
-      - VALIDATOR_PORT=30346
-      - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30346
-      - NAME=Node3
-      - BASE_PATH=/data/5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK
-      - NODE_KEY_PATH=/data/5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK/p2p_secret
+      file: docker-compose.base.yml
+      service: Node3
+    network_mode: host
 
   Node4:
     extends:
-      file: common.yml
-      service: AlephNonBootNode
-    container_name: Node4
-    environment:
-      - RPC_PORT=9937
-      - WS_PORT=9947
-      - PORT=30337
-      - VALIDATOR_PORT=30347
-      - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30347
-      - NAME=Node4
-      - BASE_PATH=/data/5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW
-      - NODE_KEY_PATH=/data/5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW/p2p_secret
+      file: docker-compose.base.yml
+      service: Node4
+    network_mode: host

From dc072d9478c7f19562b58113ef1ca3440712bef6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Fri, 4 Nov 2022 13:01:34 +0100
Subject: [PATCH 03/17] catchup_version_upgrade_test.sh

---
 .github/scripts/check_finalization.sh   |  12 +-
 .github/scripts/run_consensus.sh        |  10 +-
 scripts/catchup_version_upgrade_test.sh | 208 ++++++++++++++++++++++++
 3 files changed, 220 insertions(+), 10 deletions(-)
 create mode 100644 scripts/catchup_version_upgrade_test.sh

diff --git a/.github/scripts/check_finalization.sh b/.github/scripts/check_finalization.sh
index 36d21ad191..1611056317 100755
--- a/.github/scripts/check_finalization.sh
+++ b/.github/scripts/check_finalization.sh
@@ -1,12 +1,12 @@
 #!/bin/bash
 
-RPC_HOST=127.0.0.1
-RPC_PORT=9933
+RPC_HOST=${RPC_HOST:-127.0.0.1}
+RPC_PORT=${RPC_PORT:-9933}
 LAST_FINALIZED=""
-VALIDATOR=damian
+VALIDATOR=${VALIDATOR:-damian}
 
 while [[ "$LAST_FINALIZED" =~ "0x0" ]] || [[ -z "$LAST_FINALIZED" ]]; do
-  block_hash=$(docker run --network container:$VALIDATOR appropriate/curl:latest \
+  block_hash=$(docker run --rm --network container:$VALIDATOR appropriate/curl:latest \
                       -H "Content-Type: application/json" \
                       -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getFinalizedHead"}' http://$RPC_HOST:$RPC_PORT | jq '.result')
   ret_val=$?
@@ -15,7 +15,7 @@ while [[ "$LAST_FINALIZED" =~ "0x0" ]] || [[ -z "$LAST_FINALIZED" ]]; do
     continue
   fi
 
-  finalized_block=$(docker run --network container:$VALIDATOR appropriate/curl:latest \
+  finalized_block=$(docker run --rm --network container:$VALIDATOR appropriate/curl:latest \
                            -H "Content-Type: application/json" \
                            -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlock", "params": ['$block_hash']}' http://$RPC_HOST:$RPC_PORT | jq '.result.block.header.number')
 
@@ -25,9 +25,9 @@ while [[ "$LAST_FINALIZED" =~ "0x0" ]] || [[ -z "$LAST_FINALIZED" ]]; do
     continue
   else
     LAST_FINALIZED=$finalized_block
-    echo "Last finalized block number: $LAST_FINALIZED"
   fi
 
 done
 
+echo "Last finalized block number: $LAST_FINALIZED"
 exit $?
diff --git a/.github/scripts/run_consensus.sh b/.github/scripts/run_consensus.sh
index e8ebae3372..ca40484618 100755
--- a/.github/scripts/run_consensus.sh
+++ b/.github/scripts/run_consensus.sh
@@ -6,6 +6,7 @@ set -euo pipefail
 # change when increasing the number of node containers
 NODE_COUNT=5
 MIN_VALIDATOR_COUNT=4
+DOCKER_COMPOSE=${DOCKER_COMPOSE:-"docker/docker-compose.yml"}
 OVERRIDE_DOCKER_COMPOSE=${OVERRIDE_DOCKER_COMPOSE:-""}
 
 # default minimum validator count
@@ -83,19 +84,20 @@ function generate_bootnode_peer_id {
 
 function run_containers {
   local authorities_count="$1"
-  local override_file="$2"
+  local docker_compose_file="$2"
+  local override_file="$3"
 
   echo "Running ${authorities_count} containers..."
   if [[ -z ${override_file} ]]; then
-      docker-compose -f docker/docker-compose.yml up -d
+      docker-compose -f "${docker_compose_file}" up -d
   else
-      docker-compose -f docker/docker-compose.yml -f "${override_file}" up -d
+      docker-compose -f "${docker_compose_file}" -f "${override_file}" up -d
   fi
 }
 
 authorities=$(generate_authorities ${NODE_COUNT})
 generate_chainspec "${authorities[@]}" "${MIN_VALIDATOR_COUNT}"
 generate_bootnode_peer_id ${authorities[0]}
-run_containers ${NODE_COUNT} "${OVERRIDE_DOCKER_COMPOSE}"
+run_containers ${NODE_COUNT} "${DOCKER_COMPOSE}" "${OVERRIDE_DOCKER_COMPOSE}"
 
 exit $?
diff --git a/scripts/catchup_version_upgrade_test.sh b/scripts/catchup_version_upgrade_test.sh
new file mode 100644
index 0000000000..1d9a9b3a25
--- /dev/null
+++ b/scripts/catchup_version_upgrade_test.sh
@@ -0,0 +1,208 @@
+#!/bin/bash
+
+set -euo pipefail
+
+INIT_BLOCK=${INIT_BLOCK:-3}
+UPGRADE_ROUND=${UPGRADE_ROUND:-31}
+UPGRADE_VERSION=${UPGRADE_VERSION:-1}
+NODES=${NODES:-"Node1:Node2"}
+PORTS=${PORTS:-9934:9935}
+UPGRADE_BEFORE_DISABLE=${UPGRADE_BEFORE_DISABLE:-false}
+SEED=${SEED:-"//0"}
+ALL_NODES=${ALL_NODES:-"Node0:Node1:Node2:Node3:Node4"}
+ALL_NODES_PORTS=${ALL_NODES_PORTS:-"9933:9934:9935:9936:9937"}
+WAIT_ROUNDS=${WAIT_ROUNDS:-30}
+EXT_STATUS=${EXT_STATUS:-"in-block"}
+
+function log() {
+    echo $1 1>&2
+}
+
+function into_array() {
+    result=()
+    local tmp=$IFS
+    IFS=:
+    for e in $1; do
+        result+=($e)
+    done
+    IFS=$tmp
+}
+
+function initialize {
+    wait_for_finalized_block $1 $2 $3
+}
+
+function wait_for_finalized_block() {
+    local init_block=$1
+    local node=$2
+    local port=$3
+
+    while [[ $(get_best_finalized $node $port ) -le $init_block ]]; do
+        sleep 3
+    done
+}
+
+function get_best_finalized {
+    local validator=$1
+    local rpc_port=$2
+
+    local best_finalized=$(VALIDATOR=$validator RPC_HOST="127.0.0.1" RPC_PORT=$rpc_port ./.github/scripts/check_finalization.sh | sed 's/Last finalized block number: "\(.*\)"/\1/')
+    printf "%d" $best_finalized
+}
+
+function set_upgrade_round {
+    local round=$1
+    local version=$2
+    local validator=$3
+    local port=$4
+    local seed=$5
+    local status=$6
+
+    docker run --rm --network container:$validator cliain:latest --node 127.0.0.1:$port --seed $seed version-upgrade-schedule --version $version --session $round --expected-state $status
+}
+
+function check_if_disconnected() {
+    local -n nodes=$1
+    local -n ports=$2
+
+    log "checking if nodes are disconnected"
+
+    for i in "${!nodes[@]}"; do
+        local node=${nodes[$i]}
+        local port=${ports[$i]}
+
+        log "checking if node $node is disconnected"
+
+        last_finalized=$(get_best_finalized $node $port)
+        log "last finalized block at node $node is $last_finalized"
+
+        last_block=$(get_last_block $node $port)
+        log "last block at node $node is $last_block"
+
+        # what else we can do?
+        log "sleeping for 20 seconds"
+        sleep 20
+
+        new_finalized=$(get_best_finalized $node $port)
+        log "newest finalized at node $node after waiting is $new_finalized"
+
+        if [[ $(($new_finalized - $last_finalized)) -ge 1 ]]; then
+            log ""somehow a disconnected node $node was able to finalize new blocks""
+            exit -1
+        fi
+    done
+}
+
+function connect_nodes {
+    local -n nodes=$1
+    for node in ${nodes[@]}; do
+        docker network connect main-network $node
+    done
+}
+
+function disconnect_nodes {
+    local -n nodes=$1
+
+    for node in ${nodes[@]}; do
+        log "disconnecting node $node..."
+        docker network disconnect main-network $node
+        log "node $node disconnected"
+    done
+}
+
+function wait_for_round {
+    local round=$1
+    local validator=$2
+    local rpc_port=$3
+
+    local last_block=""
+    while [[ -z "$last_block" ]]; do
+        last_block=$(docker run --rm --network container:$validator appropriate/curl:latest \
+                            -H "Content-Type: application/json" \
+                            -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlockHash", "params": '$round'}' http://127.0.0.1:$rpc_port | jq '.result')
+    done
+}
+
+function get_last_block {
+    local validator=$1
+    local rpc_port=$2
+
+    local last_block_hash=$(docker run --rm --network container:$validator appropriate/curl:latest \
+                                   -H "Content-Type: application/json" \
+                                   -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlockHash"}' http://127.0.0.1:$rpc_port | jq '.result')
+    local last_block_number=$(docker run --rm --network container:$validator appropriate/curl:latest \
+           -H "Content-Type: application/json" \
+           -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlock", "params": ['$last_block_hash']}' http://127.0.0.1:$rpc_port | jq '.result.block.header.number')
+    printf "%d" $last_block_number
+}
+
+function check_finalization {
+    local block_to_check=$1
+    local -n nodes=$2
+    local -n ports=$3
+
+    log "checking finalization for block $block_to_check"
+
+    for i in "${!nodes[@]}"; do
+        local node=${nodes[$i]}
+        local rpc_port=${ports[$i]}
+
+        log "checking finalization at node $node"
+        wait_for_finalized_block $block_to_check $node $rpc_port
+    done
+}
+
+into_array $NODES
+NODES=(${result[@]})
+
+into_array $PORTS
+PORTS=(${result[@]})
+
+into_array $ALL_NODES
+ALL_NODES=(${result[@]})
+
+into_array "$ALL_NODES_PORTS"
+ALL_NODES_PORTS=(${result[@]})
+
+log "initializing nodes..."
+OVERRIDE_DOCKER_COMPOSE=./docker/docker-compose.bridged.yml ./.github/scripts/run_consensus.sh 1>&2
+log "awaiting finalization of $INIT_BLOCK blocks..."
+initialize $INIT_BLOCK "Node0" 9933
+log "nodes initialized"
+
+last_block=$(get_last_block "Node0" 9933)
+round_for_upgrade=$(($UPGRADE_ROUND + $last_block))
+if [[ $UPGRADE_BEFORE_DISABLE = true ]]; then
+    log "setting upgrade at $round_for_upgrade round for version $UPGRADE_VERSION before disconnecting"
+    set_upgrade_round $round_for_upgrade $UPGRADE_VERSION "Node0" 9943 $SEED $EXT_STATUS
+fi
+
+log "disconnecting nodes..."
+disconnect_nodes NODES
+log "verifying if nodes are properly disconnected..."
+check_if_disconnected NODES PORTS
+log "nodes disconnected"
+
+last_block=$(get_last_block "Node0" 9933)
+round_for_upgrade=$(($UPGRADE_ROUND + $last_block))
+if [[ $UPGRADE_BEFORE_DISABLE = false ]]; then
+    log "setting upgrade at $round_for_upgrade for version $UPGRADE_VERSION"
+    set_upgrade_round $round_for_upgrade $UPGRADE_VERSION "Node0" 9943 ${SEED} $EXT_STATUS
+fi
+
+last_block=$(get_last_block "Node0" 9933)
+awaited_round=$(($WAIT_ROUNDS+$round_for_upgrade))
+log "awaiting round $awaited_round"
+wait_for_round $awaited_round "Node0" 9933
+log "awaiting finished"
+
+log "connecting nodes..."
+connect_nodes NODES
+log "nodes connected"
+
+last_block=$(get_last_block "Node0" 9933)
+log "checking finalization..."
+check_finalization $(($awaited_round+1)) ALL_NODES ALL_NODES_PORTS
+log "finalization checked"
+
+exit $?

From 83e02be073772da1e896a1efedb0afcad8c56312 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Fri, 4 Nov 2022 13:02:05 +0100
Subject: [PATCH 04/17] added the catchup version upgrade test to github
 pipelines

---
 .github/workflows/e2e-tests-main-devnet.yml | 159 ++++++++++++++++++++
 1 file changed, 159 insertions(+)

diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml
index 94299ca870..0b632df940 100644
--- a/.github/workflows/e2e-tests-main-devnet.yml
+++ b/.github/workflows/e2e-tests-main-devnet.yml
@@ -54,6 +54,36 @@ jobs:
           retention-days: 7
 
 
+  build-cliain-image:
+    name: Build docker image for cliain
+    runs-on: ubuntu-20.04
+    steps:
+      - name: GIT | Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Install Rust toolchain
+        uses: actions-rs/toolchain@v1
+
+      - name: Cargo | Build release binary
+        run: |
+          cd ./bin/cliain && cargo build --release
+
+      - name: Build docker image
+        run: |
+          cd ./bin/cliain
+          chmod +x target/release/cliain
+          docker build --tag cliain:latest -f ./Dockerfile .
+          docker save -o cliain.tar cliain:latest
+
+      - name: Upload test docker image
+        uses: actions/upload-artifact@v2
+        with:
+          name: cliain-docker
+          path: ./bin/cliain/cliain.tar
+          if-no-files-found: error
+          retention-days: 7
+
+
   check-determinism:
     needs: [build-new-node]
     name: Verify runtime build determinism
@@ -279,6 +309,7 @@ jobs:
           test-case: fee_calculation
         timeout-minutes: 2
 
+
   run-e2e-validators-rotate:
     needs: [build-test-docker, build-test-client]
     name: Run validators rotation test
@@ -485,6 +516,130 @@ jobs:
           follow-up-finalization-check: true
         timeout-minutes: 15
 
+  run-e2e-version-upgrade-catchup-one-node-late:
+    needs: [build-test-docker, build-cliain-image]
+    name: Run a test where one of the nodes learns of version-upgrade during catchup
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Download artifact with docker image for aleph-node
+        uses: actions/download-artifact@v2
+        with:
+          name: aleph-test-docker
+
+      - name: Load node docker image
+        shell: bash
+        run: docker load -i aleph-node.tar
+
+      - name: Download artifact with docker image for cliain
+        uses: actions/download-artifact@v2
+        with:
+          name: cliain-docker
+
+      - name: Load cliain docker image
+        shell: bash
+        run: docker load -i cliain.tar
+
+      - name: Call catchup_test.sh
+        run: |
+          chmod +x ./scripts/catchup_version_upgrade_test.sh
+          UPGRADE_ROUND=31 NODES="Node1" PORTS="9934" EXT_STATUS="finalized" DOCKER_COMPOSE=docker/docker-compose.bridged.yml ./scripts/catchup_version_upgrade_test.sh
+
+  run-e2e-version-upgrade-catchup-two-nodes-late:
+    needs: [build-test-docker, build-cliain-image]
+    name: Run a test where two nodes learn of version-upgrade during catchup
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Download artifact with docker image for aleph-node
+        uses: actions/download-artifact@v2
+        with:
+          name: aleph-test-docker
+
+      - name: Load node docker image
+        shell: bash
+        run: docker load -i aleph-node.tar
+
+      - name: Download artifact with docker image for cliain
+        uses: actions/download-artifact@v2
+        with:
+          name: cliain-docker
+
+      - name: Load cliain docker image
+        shell: bash
+        run: docker load -i cliain.tar
+
+      - name: Call catchup_test.sh
+        run: |
+          chmod +x ./scripts/catchup_version_upgrade_test.sh
+          UPGRADE_ROUND=31 NODES="Node1:Node2" PORTS="9934:9935" EXT_STATUS="in-block" DOCKER_COMPOSE=docker/docker-compose.bridged.yml ./scripts/catchup_version_upgrade_test.sh
+
+  run-e2e-version-upgrade-early-catchup-one-node-late:
+    needs: [build-test-docker, build-cliain-image]
+    name: Run a test where one of the nodes learns of version-upgrade before catchup
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Download artifact with docker image for aleph-node
+        uses: actions/download-artifact@v2
+        with:
+          name: aleph-test-docker
+
+      - name: Load node docker image
+        shell: bash
+        run: docker load -i aleph-node.tar
+
+      - name: Download artifact with docker image for cliain
+        uses: actions/download-artifact@v2
+        with:
+          name: cliain-docker
+
+      - name: Load cliain docker image
+        shell: bash
+        run: docker load -i cliain.tar
+
+      - name: Call catchup_test.sh
+        run: |
+          chmod +x ./scripts/catchup_version_upgrade_test.sh
+          UPGRADE_BEFORE_DISABLE=true UPGRADE_ROUND=31 NODES="Node1" PORTS="9934" EXT_STATUS="finalized" DOCKER_COMPOSE=docker/docker-compose.bridged.yml ./scripts/catchup_version_upgrade_test.sh
+
+  run-e2e-version-upgrade-early-catchup-two-nodes-late:
+    needs: [build-test-docker, build-cliain-image]
+    name: Run a test where two nodes learn of version-upgrade before catchup
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Download artifact with docker image for aleph-node
+        uses: actions/download-artifact@v2
+        with:
+          name: aleph-test-docker
+
+      - name: Load node docker image
+        shell: bash
+        run: docker load -i aleph-node.tar
+
+      - name: Download artifact with docker image for cliain
+        uses: actions/download-artifact@v2
+        with:
+          name: cliain-docker
+
+      - name: Load cliain docker image
+        shell: bash
+        run: docker load -i cliain.tar
+
+      - name: Call catchup_test.sh
+        run: |
+          chmod +x ./scripts/catchup_version_upgrade_test.sh
+          UPGRADE_BEFORE_DISABLE=true UPGRADE_ROUND=31 NODES="Node1:Node2" PORTS="9934:9935" EXT_STATUS="finalized" DOCKER_COMPOSE=docker/docker-compose.bridged.yml ./scripts/catchup_version_upgrade_test.sh
+
   check-e2e-test-suite-completion:
     needs: [
       run-e2e-finalization-test,
@@ -510,6 +665,10 @@ jobs:
       run-e2e-ban-manual,
       run-e2e-version-upgrade,
       run-e2e-failing-version-upgrade,
+      run-e2e-version-upgrade-catchup-one-node-late,
+      run-e2e-version-upgrade-catchup-two-nodes-late,
+      run-e2e-version-upgrade-early-catchup-one-node-late,
+      run-e2e-version-upgrade-early-catchup-two-nodes-late,
     ]
     name: Check e2e test suite completion
     runs-on: ubuntu-20.04

From 143ec1bb4a0f1f0e63fc5e2a92a9760417983328 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Sun, 6 Nov 2022 21:22:32 +0100
Subject: [PATCH 05/17] review changes

---
 scripts/catchup_version_upgrade_test.sh | 51 ++++++++++++-------------
 1 file changed, 24 insertions(+), 27 deletions(-)

diff --git a/scripts/catchup_version_upgrade_test.sh b/scripts/catchup_version_upgrade_test.sh
index 1d9a9b3a25..fd2e71eb6f 100644
--- a/scripts/catchup_version_upgrade_test.sh
+++ b/scripts/catchup_version_upgrade_test.sh
@@ -3,7 +3,7 @@
 set -euo pipefail
 
 INIT_BLOCK=${INIT_BLOCK:-3}
-UPGRADE_ROUND=${UPGRADE_ROUND:-31}
+UPGRADE_BLOCK=${UPGRADE_BLOCK:-31}
 UPGRADE_VERSION=${UPGRADE_VERSION:-1}
 NODES=${NODES:-"Node1:Node2"}
 PORTS=${PORTS:-9934:9935}
@@ -11,7 +11,7 @@ UPGRADE_BEFORE_DISABLE=${UPGRADE_BEFORE_DISABLE:-false}
 SEED=${SEED:-"//0"}
 ALL_NODES=${ALL_NODES:-"Node0:Node1:Node2:Node3:Node4"}
 ALL_NODES_PORTS=${ALL_NODES_PORTS:-"9933:9934:9935:9936:9937"}
-WAIT_ROUNDS=${WAIT_ROUNDS:-30}
+WAIT_BLOCKS=${WAIT_BLOCKS:-30}
 EXT_STATUS=${EXT_STATUS:-"in-block"}
 
 function log() {
@@ -37,7 +37,7 @@ function wait_for_finalized_block() {
     local node=$2
     local port=$3
 
-    while [[ $(get_best_finalized $node $port ) -le $init_block ]]; do
+    while [[ $(get_best_finalized $node $port) -le $init_block ]]; do
         sleep 3
     done
 }
@@ -50,15 +50,15 @@ function get_best_finalized {
     printf "%d" $best_finalized
 }
 
-function set_upgrade_round {
-    local round=$1
+function set_upgrade_session {
+    local session=$1
     local version=$2
     local validator=$3
     local port=$4
     local seed=$5
     local status=$6
 
-    docker run --rm --network container:$validator cliain:latest --node 127.0.0.1:$port --seed $seed version-upgrade-schedule --version $version --session $round --expected-state $status
+    docker run --rm --network container:$validator cliain:latest --node 127.0.0.1:$port --seed $seed version-upgrade-schedule --version $version --session $session --expected-state $status
 }
 
 function check_if_disconnected() {
@@ -84,10 +84,10 @@ function check_if_disconnected() {
         sleep 20
 
         new_finalized=$(get_best_finalized $node $port)
-        log "newest finalized at node $node after waiting is $new_finalized"
+        log "newest finalized block at node $node after waiting is $new_finalized"
 
         if [[ $(($new_finalized - $last_finalized)) -ge 1 ]]; then
-            log ""somehow a disconnected node $node was able to finalize new blocks""
+            log "somehow a disconnected node $node was able to finalize new blocks"
             exit -1
         fi
     done
@@ -110,8 +110,8 @@ function disconnect_nodes {
     done
 }
 
-function wait_for_round {
-    local round=$1
+function wait_for_block {
+    local block=$1
     local validator=$2
     local rpc_port=$3
 
@@ -119,7 +119,7 @@ function wait_for_round {
     while [[ -z "$last_block" ]]; do
         last_block=$(docker run --rm --network container:$validator appropriate/curl:latest \
                             -H "Content-Type: application/json" \
-                            -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlockHash", "params": '$round'}' http://127.0.0.1:$rpc_port | jq '.result')
+                            -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlockHash", "params": '$block'}' http://127.0.0.1:$rpc_port | jq '.result')
     done
 }
 
@@ -127,12 +127,9 @@ function get_last_block {
     local validator=$1
     local rpc_port=$2
 
-    local last_block_hash=$(docker run --rm --network container:$validator appropriate/curl:latest \
-                                   -H "Content-Type: application/json" \
-                                   -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlockHash"}' http://127.0.0.1:$rpc_port | jq '.result')
     local last_block_number=$(docker run --rm --network container:$validator appropriate/curl:latest \
-           -H "Content-Type: application/json" \
-           -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlock", "params": ['$last_block_hash']}' http://127.0.0.1:$rpc_port | jq '.result.block.header.number')
+                                     -H "Content-Type: application/json" \
+                                     -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlock"}' http://127.0.0.1:$rpc_port | jq '.result.block.header.number')
     printf "%d" $last_block_number
 }
 
@@ -165,16 +162,16 @@ into_array "$ALL_NODES_PORTS"
 ALL_NODES_PORTS=(${result[@]})
 
 log "initializing nodes..."
-OVERRIDE_DOCKER_COMPOSE=./docker/docker-compose.bridged.yml ./.github/scripts/run_consensus.sh 1>&2
+DOCKER_COMPOSE=./docker/docker-compose.bridged.yml ./.github/scripts/run_consensus.sh 1>&2
 log "awaiting finalization of $INIT_BLOCK blocks..."
 initialize $INIT_BLOCK "Node0" 9933
 log "nodes initialized"
 
 last_block=$(get_last_block "Node0" 9933)
-round_for_upgrade=$(($UPGRADE_ROUND + $last_block))
+block_for_upgrade=$(($UPGRADE_BLOCK + $last_block))
 if [[ $UPGRADE_BEFORE_DISABLE = true ]]; then
-    log "setting upgrade at $round_for_upgrade round for version $UPGRADE_VERSION before disconnecting"
-    set_upgrade_round $round_for_upgrade $UPGRADE_VERSION "Node0" 9943 $SEED $EXT_STATUS
+    log "setting upgrade at $block_for_upgrade block for version $UPGRADE_VERSION before disconnecting"
+    set_upgrade_session $block_for_upgrade $UPGRADE_VERSION "Node0" 9943 $SEED $EXT_STATUS
 fi
 
 log "disconnecting nodes..."
@@ -184,16 +181,16 @@ check_if_disconnected NODES PORTS
 log "nodes disconnected"
 
 last_block=$(get_last_block "Node0" 9933)
-round_for_upgrade=$(($UPGRADE_ROUND + $last_block))
+block_for_upgrade=$(($UPGRADE_BLOCK + $last_block))
 if [[ $UPGRADE_BEFORE_DISABLE = false ]]; then
-    log "setting upgrade at $round_for_upgrade for version $UPGRADE_VERSION"
-    set_upgrade_round $round_for_upgrade $UPGRADE_VERSION "Node0" 9943 ${SEED} $EXT_STATUS
+    log "setting upgrade at $block_for_upgrade block for version $UPGRADE_VERSION"
+    set_upgrade_session $block_for_upgrade $UPGRADE_VERSION "Node0" 9943 ${SEED} $EXT_STATUS
 fi
 
 last_block=$(get_last_block "Node0" 9933)
-awaited_round=$(($WAIT_ROUNDS+$round_for_upgrade))
-log "awaiting round $awaited_round"
-wait_for_round $awaited_round "Node0" 9933
+awaited_block=$(($WAIT_BLOCKS+$block_for_upgrade))
+log "awaiting block $awaited_block"
+wait_for_block $awaited_block "Node0" 9933
 log "awaiting finished"
 
 log "connecting nodes..."
@@ -202,7 +199,7 @@ log "nodes connected"
 
 last_block=$(get_last_block "Node0" 9933)
 log "checking finalization..."
-check_finalization $(($awaited_round+1)) ALL_NODES ALL_NODES_PORTS
+check_finalization $(($awaited_block+1)) ALL_NODES ALL_NODES_PORTS
 log "finalization checked"
 
 exit $?

From 8ca57d733a8ba2d411352f27a2120df6806e1337 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Mon, 7 Nov 2022 12:40:46 +0100
Subject: [PATCH 06/17] chmod +x catchup_version_upgrade_test.sh

---
 scripts/catchup_version_upgrade_test.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 scripts/catchup_version_upgrade_test.sh

diff --git a/scripts/catchup_version_upgrade_test.sh b/scripts/catchup_version_upgrade_test.sh
old mode 100644
new mode 100755

From 02fd76bd1e0eafd1f892d78ad06f1fe88d539f9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Mon, 7 Nov 2022 12:49:19 +0100
Subject: [PATCH 07/17] review changes: using `matrix` for catch-up version
 upgrade test

---
 .github/workflows/e2e-tests-main-devnet.yml | 132 +++++---------------
 1 file changed, 30 insertions(+), 102 deletions(-)

diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml
index 0b632df940..80a40f8474 100644
--- a/.github/workflows/e2e-tests-main-devnet.yml
+++ b/.github/workflows/e2e-tests-main-devnet.yml
@@ -64,14 +64,20 @@ jobs:
       - name: Install Rust toolchain
         uses: actions-rs/toolchain@v1
 
+      - name: Restore cache
+        uses: .github/actions/restore-cache
+        with:
+          target-key: cliain
+          cache-version: v2
+          cargo-targets: bin/cliain/target/
+
       - name: Cargo | Build release binary
         run: |
-          cd ./bin/cliain && cargo build --release
+          cd bin/cliain && cargo build --release
 
       - name: Build docker image
         run: |
-          cd ./bin/cliain
-          chmod +x target/release/cliain
+          cd bin/cliain
           docker build --tag cliain:latest -f ./Dockerfile .
           docker save -o cliain.tar cliain:latest
 
@@ -516,72 +522,21 @@ jobs:
           follow-up-finalization-check: true
         timeout-minutes: 15
 
-  run-e2e-version-upgrade-catchup-one-node-late:
-    needs: [build-test-docker, build-cliain-image]
-    name: Run a test where one of the nodes learns of version-upgrade during catchup
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout source code
-        uses: actions/checkout@v2
-
-      - name: Download artifact with docker image for aleph-node
-        uses: actions/download-artifact@v2
-        with:
-          name: aleph-test-docker
-
-      - name: Load node docker image
-        shell: bash
-        run: docker load -i aleph-node.tar
-
-      - name: Download artifact with docker image for cliain
-        uses: actions/download-artifact@v2
-        with:
-          name: cliain-docker
-
-      - name: Load cliain docker image
-        shell: bash
-        run: docker load -i cliain.tar
-
-      - name: Call catchup_test.sh
-        run: |
-          chmod +x ./scripts/catchup_version_upgrade_test.sh
-          UPGRADE_ROUND=31 NODES="Node1" PORTS="9934" EXT_STATUS="finalized" DOCKER_COMPOSE=docker/docker-compose.bridged.yml ./scripts/catchup_version_upgrade_test.sh
-
-  run-e2e-version-upgrade-catchup-two-nodes-late:
+  run-e2e-version-upgrade-catchup:
     needs: [build-test-docker, build-cliain-image]
-    name: Run a test where two nodes learn of version-upgrade during catchup
+    name: Run a series of tests where one of the nodes learns of version-upgrade during catchup
     runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout source code
-        uses: actions/checkout@v2
-
-      - name: Download artifact with docker image for aleph-node
-        uses: actions/download-artifact@v2
-        with:
-          name: aleph-test-docker
-
-      - name: Load node docker image
-        shell: bash
-        run: docker load -i aleph-node.tar
-
-      - name: Download artifact with docker image for cliain
-        uses: actions/download-artifact@v2
-        with:
-          name: cliain-docker
+    strategy:
+      matrix:
+        upgrade_before_disable: ["true", "false"]
 
-      - name: Load cliain docker image
-        shell: bash
-        run: docker load -i cliain.tar
-
-      - name: Call catchup_test.sh
-        run: |
-          chmod +x ./scripts/catchup_version_upgrade_test.sh
-          UPGRADE_ROUND=31 NODES="Node1:Node2" PORTS="9934:9935" EXT_STATUS="in-block" DOCKER_COMPOSE=docker/docker-compose.bridged.yml ./scripts/catchup_version_upgrade_test.sh
-
-  run-e2e-version-upgrade-early-catchup-one-node-late:
-    needs: [build-test-docker, build-cliain-image]
-    name: Run a test where one of the nodes learns of version-upgrade before catchup
-    runs-on: ubuntu-20.04
+        include:
+          - nodes: "Node1"
+            ports: "9934"
+            ext_status: "finalized"
+          - nodes: "Node1:NODE2"
+            ports: "9934:9935"
+            ext_status: "in-block"
     steps:
       - name: Checkout source code
         uses: actions/checkout@v2
@@ -605,40 +560,16 @@ jobs:
         run: docker load -i cliain.tar
 
       - name: Call catchup_test.sh
-        run: |
-          chmod +x ./scripts/catchup_version_upgrade_test.sh
-          UPGRADE_BEFORE_DISABLE=true UPGRADE_ROUND=31 NODES="Node1" PORTS="9934" EXT_STATUS="finalized" DOCKER_COMPOSE=docker/docker-compose.bridged.yml ./scripts/catchup_version_upgrade_test.sh
-
-  run-e2e-version-upgrade-early-catchup-two-nodes-late:
-    needs: [build-test-docker, build-cliain-image]
-    name: Run a test where two nodes learn of version-upgrade before catchup
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout source code
-        uses: actions/checkout@v2
-
-      - name: Download artifact with docker image for aleph-node
-        uses: actions/download-artifact@v2
-        with:
-          name: aleph-test-docker
-
-      - name: Load node docker image
-        shell: bash
-        run: docker load -i aleph-node.tar
-
-      - name: Download artifact with docker image for cliain
-        uses: actions/download-artifact@v2
-        with:
-          name: cliain-docker
-
-      - name: Load cliain docker image
-        shell: bash
-        run: docker load -i cliain.tar
+        env:
+          UPGRADE_BLOCK: 31
+          NODES: ${{ matrix.nodes }}
+          PORTS: ${{ matrix.ports }}
+          EXT_STATUS: ${{ matrix.ext_status }}
+          UPGRADE_BEFORE_DISABLE: ${{ matrix.upgrade_before_disable }}
+          DOCKER_COMPOSE: docker/docker-compose.bridged.yml
 
-      - name: Call catchup_test.sh
         run: |
-          chmod +x ./scripts/catchup_version_upgrade_test.sh
-          UPGRADE_BEFORE_DISABLE=true UPGRADE_ROUND=31 NODES="Node1:Node2" PORTS="9934:9935" EXT_STATUS="finalized" DOCKER_COMPOSE=docker/docker-compose.bridged.yml ./scripts/catchup_version_upgrade_test.sh
+          ./scripts/catchup_version_upgrade_test.sh
 
   check-e2e-test-suite-completion:
     needs: [
@@ -665,10 +596,7 @@ jobs:
       run-e2e-ban-manual,
       run-e2e-version-upgrade,
       run-e2e-failing-version-upgrade,
-      run-e2e-version-upgrade-catchup-one-node-late,
-      run-e2e-version-upgrade-catchup-two-nodes-late,
-      run-e2e-version-upgrade-early-catchup-one-node-late,
-      run-e2e-version-upgrade-early-catchup-two-nodes-late,
+      run-e2e-version-upgrade-catchup,
     ]
     name: Check e2e test suite completion
     runs-on: ubuntu-20.04

From 87defd79845e8b614a42186deb464247112815b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Mon, 7 Nov 2022 12:50:22 +0100
Subject: [PATCH 08/17] review changes: bumped version of aleph-client; added
 schedule_version_upgrade_with_state

---
 aleph-client/Cargo.toml             |  2 +-
 aleph-client/src/lib.rs             |  2 +-
 aleph-client/src/version_upgrade.rs | 10 +++++++++-
 bin/cliain/src/version_upgrade.rs   |  4 ++--
 4 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/aleph-client/Cargo.toml b/aleph-client/Cargo.toml
index 7f71b82ebe..9439f1960a 100644
--- a/aleph-client/Cargo.toml
+++ b/aleph-client/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "aleph_client"
-version = "1.11.0"
+version = "1.12.0"
 edition = "2021"
 license = "Apache 2.0"
 
diff --git a/aleph-client/src/lib.rs b/aleph-client/src/lib.rs
index cca7702d24..6b327c5350 100644
--- a/aleph-client/src/lib.rs
+++ b/aleph-client/src/lib.rs
@@ -57,7 +57,7 @@ pub use treasury::{
     propose as make_treasury_proposal, reject as reject_treasury_proposal, staking_treasury_payout,
     treasury_account,
 };
-pub use version_upgrade::{schedule_upgrade, Version};
+pub use version_upgrade::{schedule_upgrade, schedule_upgrade_with_state, Version};
 pub use vesting::{
     get_schedules, merge_schedules, vest, vest_other, vested_transfer, VestingError,
     VestingSchedule,
diff --git a/aleph-client/src/version_upgrade.rs b/aleph-client/src/version_upgrade.rs
index bc3150f819..f180d5437d 100644
--- a/aleph-client/src/version_upgrade.rs
+++ b/aleph-client/src/version_upgrade.rs
@@ -6,7 +6,7 @@ use crate::{try_send_xt, AnyConnection, RootConnection};
 
 pub type Version = u32;
 
-pub fn schedule_upgrade(
+pub fn schedule_upgrade_with_state(
     connection: &RootConnection,
     version: Version,
     session: SessionIndex,
@@ -35,3 +35,11 @@ pub fn schedule_upgrade(
     )
     .map(|_| ())
 }
+
+pub fn schedule_upgrade(
+    connection: &RootConnection,
+    version: Version,
+    session: SessionIndex,
+) -> Result<(), ApiClientError> {
+    schedule_upgrade_with_state(connection, version, session, XtStatus::Finalized)
+}
diff --git a/bin/cliain/src/version_upgrade.rs b/bin/cliain/src/version_upgrade.rs
index a38cf8913e..9f8b07f868 100644
--- a/bin/cliain/src/version_upgrade.rs
+++ b/bin/cliain/src/version_upgrade.rs
@@ -1,4 +1,4 @@
-use aleph_client::RootConnection;
+use aleph_client::{schedule_upgrade_with_state, RootConnection};
 use anyhow::Error;
 use primitives::SessionIndex;
 
@@ -10,7 +10,7 @@ pub fn schedule_upgrade(
     session_for_upgrade: SessionIndex,
     expected_state: ExtrinsicState,
 ) -> anyhow::Result<()> {
-    aleph_client::schedule_upgrade(
+    schedule_upgrade_with_state(
         &connection,
         version,
         session_for_upgrade,

From 65a41f689e332d108f44800b1a2496de83beeeff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Mon, 7 Nov 2022 12:55:00 +0100
Subject: [PATCH 09/17] review changes: comment about key derivation in
 docker-compose `//1` etc.

---
 docker/docker-compose.base.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docker/docker-compose.base.yml b/docker/docker-compose.base.yml
index c2957f7be7..6a159a6f19 100644
--- a/docker/docker-compose.base.yml
+++ b/docker/docker-compose.base.yml
@@ -26,6 +26,7 @@ services:
       - VALIDATOR_PORT=30344
       - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30344
       - NAME=Node1
+      # key derived from "//1"
       - BASE_PATH=/data/5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o
       - NODE_KEY_PATH=/data/5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o/p2p_secret
 
@@ -41,6 +42,7 @@ services:
       - VALIDATOR_PORT=30345
       - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30345
       - NAME=Node2
+      # key derived from "//2"
       - BASE_PATH=/data/5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9
       - NODE_KEY_PATH=/data/5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9/p2p_secret
 
@@ -56,6 +58,7 @@ services:
       - VALIDATOR_PORT=30346
       - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30346
       - NAME=Node3
+      # key derived from "//3"
       - BASE_PATH=/data/5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK
       - NODE_KEY_PATH=/data/5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK/p2p_secret
 
@@ -71,5 +74,6 @@ services:
       - VALIDATOR_PORT=30347
       - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30347
       - NAME=Node4
+      # key derived from "//4"
       - BASE_PATH=/data/5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW
       - NODE_KEY_PATH=/data/5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW/p2p_secret

From 4e059724981ff5bff931ed345bfe1c4a4ffec86f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Mon, 7 Nov 2022 12:58:02 +0100
Subject: [PATCH 10/17] fix: e2e...yml: wrong `uses`

---
 .github/workflows/e2e-tests-main-devnet.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml
index 80a40f8474..8e0fd2b52a 100644
--- a/.github/workflows/e2e-tests-main-devnet.yml
+++ b/.github/workflows/e2e-tests-main-devnet.yml
@@ -65,7 +65,7 @@ jobs:
         uses: actions-rs/toolchain@v1
 
       - name: Restore cache
-        uses: .github/actions/restore-cache
+        uses: ./.github/actions/restore-cache
         with:
           target-key: cliain
           cache-version: v2

From 148439c3babfced65d38691b90d1481da3ca0b7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Mon, 7 Nov 2022 13:03:55 +0100
Subject: [PATCH 11/17] bumped e2e-tests/Cargo.lock

---
 e2e-tests/Cargo.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/e2e-tests/Cargo.lock b/e2e-tests/Cargo.lock
index a5f686c588..30eb393cb1 100644
--- a/e2e-tests/Cargo.lock
+++ b/e2e-tests/Cargo.lock
@@ -121,7 +121,7 @@ dependencies = [
 
 [[package]]
 name = "aleph_client"
-version = "1.11.0"
+version = "1.12.0"
 dependencies = [
  "ac-node-api",
  "ac-primitives",

From d3d65ed2f6cf40176a13212c6e2f6899dc310609 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Mon, 7 Nov 2022 13:04:24 +0100
Subject: [PATCH 12/17] fix: version_upgrade after api refactor

---
 e2e-tests/src/test/version_upgrade.rs | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/e2e-tests/src/test/version_upgrade.rs b/e2e-tests/src/test/version_upgrade.rs
index b6231c4823..1dbd3a5193 100644
--- a/e2e-tests/src/test/version_upgrade.rs
+++ b/e2e-tests/src/test/version_upgrade.rs
@@ -1,6 +1,6 @@
 use aleph_client::{
     get_current_session, get_session_period, schedule_upgrade, wait_for_at_least_session,
-    wait_for_finalized_block, AnyConnection, XtStatus,
+    wait_for_finalized_block, AnyConnection,
 };
 use primitives::{Header, SessionIndex};
 
@@ -28,12 +28,7 @@ pub fn schedule_version_change(config: &Config) -> anyhow::Result<()> {
         .unwrap_or(UPGRADE_FINALIZATION_WAIT_SESSIONS);
     let session_after_upgrade = session_for_upgrade + wait_sessions_after_upgrade;
 
-    schedule_upgrade(
-        &connection,
-        version_for_upgrade,
-        session_for_upgrade,
-        XtStatus::Finalized,
-    )?;
+    schedule_upgrade(&connection, version_for_upgrade, session_for_upgrade)?;
 
     wait_for_at_least_session(&connection, session_after_upgrade)?;
     let block_number = session_after_upgrade * get_session_period(&connection);
@@ -61,12 +56,7 @@ pub fn schedule_doomed_version_change_and_verify_finalization_stopped(
         .unwrap_or(UPGRADE_FINALIZATION_WAIT_SESSIONS);
     let session_after_upgrade = session_for_upgrade + wait_sessions_after_upgrade;
 
-    schedule_upgrade(
-        &connection,
-        version_for_upgrade,
-        session_for_upgrade,
-        XtStatus::Finalized,
-    )?;
+    schedule_upgrade(&connection, version_for_upgrade, session_for_upgrade)?;
     wait_for_at_least_session(&connection, session_for_upgrade)?;
     let last_finalized_block = session_for_upgrade * get_session_period(&connection) - 1;
 

From 35563c75ec658d2a7bd8e2dad0571f1499e68409 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Mon, 7 Nov 2022 13:24:57 +0100
Subject: [PATCH 13/17] review changes: added cache cleanup to
 build-cliain-image job

---
 .github/workflows/e2e-tests-main-devnet.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml
index 8e0fd2b52a..d3c385dc11 100644
--- a/.github/workflows/e2e-tests-main-devnet.yml
+++ b/.github/workflows/e2e-tests-main-devnet.yml
@@ -89,6 +89,9 @@ jobs:
           if-no-files-found: error
           retention-days: 7
 
+      - name: Cleanup cache
+        uses: ./.github/actions/post-cache
+
 
   check-determinism:
     needs: [build-new-node]

From 22dce1e887fea7341586287be21f48855b4dee1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Mon, 7 Nov 2022 14:18:09 +0100
Subject: [PATCH 14/17] fix: typo in e2e...yml s/NODE2/Node2

---
 .github/workflows/e2e-tests-main-devnet.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml
index d3c385dc11..b677bd9cb7 100644
--- a/.github/workflows/e2e-tests-main-devnet.yml
+++ b/.github/workflows/e2e-tests-main-devnet.yml
@@ -537,7 +537,7 @@ jobs:
           - nodes: "Node1"
             ports: "9934"
             ext_status: "finalized"
-          - nodes: "Node1:NODE2"
+          - nodes: "Node1:Node2"
             ports: "9934:9935"
             ext_status: "in-block"
     steps:

From 781c8815e9038fcbe61b10b0b834a87143ee949f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Mon, 7 Nov 2022 15:17:26 +0100
Subject: [PATCH 15/17] fix for e2e...yml: fixed matrix-include for catch-up
 version upgrade test

---
 .github/workflows/e2e-tests-main-devnet.yml | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml
index b677bd9cb7..a1859c6839 100644
--- a/.github/workflows/e2e-tests-main-devnet.yml
+++ b/.github/workflows/e2e-tests-main-devnet.yml
@@ -527,19 +527,30 @@ jobs:
 
   run-e2e-version-upgrade-catchup:
     needs: [build-test-docker, build-cliain-image]
-    name: Run a series of tests where one of the nodes learns of version-upgrade during catchup
+    name: Run series of tests where some of the nodes need to do version-upgrade during catch-up
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        upgrade_before_disable: ["true", "false"]
-
         include:
           - nodes: "Node1"
             ports: "9934"
             ext_status: "finalized"
+            upgrade_before_disable: "true"
+
+          - nodes: "Node1"
+            ports: "9934"
+            ext_status: "finalized"
+            upgrade_before_disable: "false"
+
+          - nodes: "Node1:Node2"
+            ports: "9934:9935"
+            ext_status: "in-block"
+            upgrade_before_disable: "true"
+
           - nodes: "Node1:Node2"
             ports: "9934:9935"
             ext_status: "in-block"
+            upgrade_before_disable: "false"
     steps:
       - name: Checkout source code
         uses: actions/checkout@v2

From 992926a313bd194e1f330d7d29a88461ee3dacc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Tue, 8 Nov 2022 10:29:59 +0100
Subject: [PATCH 16/17] review changes: renamed local variable in
 catchup_version_upgrade.sh

---
 scripts/catchup_version_upgrade_test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/catchup_version_upgrade_test.sh b/scripts/catchup_version_upgrade_test.sh
index fd2e71eb6f..81eaffb724 100755
--- a/scripts/catchup_version_upgrade_test.sh
+++ b/scripts/catchup_version_upgrade_test.sh
@@ -33,11 +33,11 @@ function initialize {
 }
 
 function wait_for_finalized_block() {
-    local init_block=$1
+    local block_to_be_finalized=$1
     local node=$2
     local port=$3
 
-    while [[ $(get_best_finalized $node $port) -le $init_block ]]; do
+    while [[ $(get_best_finalized $node $port) -le $block_to_be_finalized ]]; do
         sleep 3
     done
 }

From 0b2755730f8154f715e7691e91c6d8d2155d3670 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Lachowski?= <lukasz.lachowski@cardinals.cc>
Date: Tue, 8 Nov 2022 12:35:29 +0100
Subject: [PATCH 17/17] more review changes for catchup_version_upgrade_test.sh

---
 scripts/catchup_version_upgrade_test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/catchup_version_upgrade_test.sh b/scripts/catchup_version_upgrade_test.sh
index 81eaffb724..baeb1a3c5b 100755
--- a/scripts/catchup_version_upgrade_test.sh
+++ b/scripts/catchup_version_upgrade_test.sh
@@ -184,7 +184,7 @@ last_block=$(get_last_block "Node0" 9933)
 block_for_upgrade=$(($UPGRADE_BLOCK + $last_block))
 if [[ $UPGRADE_BEFORE_DISABLE = false ]]; then
     log "setting upgrade at $block_for_upgrade block for version $UPGRADE_VERSION"
-    set_upgrade_session $block_for_upgrade $UPGRADE_VERSION "Node0" 9943 ${SEED} $EXT_STATUS
+    set_upgrade_session $block_for_upgrade $UPGRADE_VERSION "Node0" 9943 $SEED $EXT_STATUS
 fi
 
 last_block=$(get_last_block "Node0" 9933)