vmware: improve solidfire storage plugin integration and fix cases (#3)…

.asf.yaml

@@ -53,7 +53,11 @@ github:
     - rajujith
     - alexandremattioli
     - vishesh92
-    - soreana
+    - GaOrtiga
     - acs-robot
+    - BryanMLima
+    - SadiJr
+    - JoaoJandre
+    - winterhazel
 
   protected_branches: ~

.github/linters/.flake8

@@ -15,8 +15,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# E112 Expected an indented block
+# E113 Unexpected indentation
+# E133 Closing bracket is missing indentation
 # E223 Tab before operator
 # E224 Tab after operator
+# E227 Missing whitespace around bitwise or shift operator
 # E242 Tab after ','
 # E273 Tab after keyword
 # E274 Tab before keyword
@@ -30,4 +34,7 @@
 # W391 Blank line at end of file
 
 [flake8]
-select = E223,E224,E242,E273,E274,E742,E743,E901,E902,W291,W292,W293,W391
+exclude =
+    .git,
+    venv
+select = E112,E113,E133,E223,E224,E227,E242,E273,E274,E742,E743,E901,E902,W291,W292,W293,W391

.github/workflows/build.yml

@@ -27,7 +27,7 @@ jobs:
   build:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up JDK 11
         uses: actions/setup-java@v3

.github/workflows/ci.yml

@@ -59,7 +59,8 @@ jobs:
                   smoke/test_domain_network_offerings
                   smoke/test_domain_service_offerings
                   smoke/test_domain_vpc_offerings",
-                "smoke/test_dynamicroles
+                "smoke/test_cluster_drs
+                  smoke/test_dynamicroles
                   smoke/test_enable_account_settings_for_domain
                   smoke/test_enable_role_based_users_in_projects
                   smoke/test_events_resource
@@ -192,7 +193,9 @@ jobs:
                 "component/test_vpc_network_lbrules" ]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Set up JDK
         uses: actions/setup-java@v3
@@ -217,6 +220,11 @@ jobs:
         run: |
           python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid
 
+      - name: Install jacoco dependencies
+        run: |
+          wget https://github.com/jacoco/jacoco/releases/download/v0.8.10/jacoco-0.8.10.zip
+          unzip jacoco-0.8.10.zip -d jacoco
+
       - name: Env details
         run: |
           uname -a
@@ -249,9 +257,16 @@ jobs:
           mvn -q -Pdeveloper -pl developer -Ddeploydb
           mvn -q -Pdeveloper -pl developer -Ddeploydb-simulator
 
+      - name: Generate jacoco-coverage.sh
+        run: |
+          echo "java -jar jacoco/lib/jacococli.jar report jacoco-it.exec \\" > jacoco-report.sh
+          find . | grep "target/classes" | sed 's/\/classes\//\/classes /g' | awk '{print "--classfiles", $1, "\\"}' | sort |uniq >> jacoco-report.sh
+          find . | grep "src/main/java" | sed 's/\/java\//\/java /g' | awk '{print "--sourcefiles", $1, "\\"}' | sort | uniq >> jacoco-report.sh
+          echo "--xml jacoco-coverage.xml" >> jacoco-report.sh
+
       - name: Start CloudStack Management Server with Simulator
         run: |
-          export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/urandom"
+          export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/urandom  -javaagent:jacoco/lib/jacocoagent.jar=address=*,port=36320,output=tcpserver"
           echo -e "\nStarting simulator"
           set +e
           mvn -Dsimulator -Dorg.eclipse.jetty.annotations.maxWait=120 -pl :cloud-client-ui jetty:run 2>&1 > /tmp/jetty-log || true &
@@ -275,10 +290,20 @@ jobs:
           done
 
           echo -e "Stopping Simulator, integration tests run completed\n"
+          java -jar jacoco/lib/jacococli.jar dump --address localhost --port 36320 --destfile jacoco-it.exec
+          bash jacoco-report.sh
           mvn -Dsimulator -pl client jetty:stop 2>&1
           find /tmp//MarvinLogs -type f -exec echo -e "Printing marvin logs {} :\n" \; -exec cat {} \;
 
       - name: Integration Tests Result
         run: |
           echo -e "Simulator CI Test Results: (only failures listed)\n"
           python3 ./tools/marvin/xunit-reader.py integration-test-results/
+
+      - uses: codecov/codecov-action@v3
+        with:
+          files: jacoco-coverage.xml
+          fail_ci_if_error: true
+          flags: simulator-marvin-tests
+          verbose: true
+          name: codecov

.github/workflows/codecov.yml

@@ -32,7 +32,7 @@ jobs:
     name: codecov
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -53,5 +53,6 @@ jobs:
         with:
           files: ./client/target/site/jacoco-aggregate/jacoco.xml
           fail_ci_if_error: true
+          flags: unit-tests
           verbose: true
           name: codecov

.github/workflows/docker-cloudstack-simulator.yml

@@ -47,7 +47,7 @@ jobs:
       - name: Set Docker repository name
         run: echo "DOCKER_REPOSITORY=apache" >> $GITHUB_ENV
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set ACS version
         run: echo "ACS_VERSION=$(grep '<version>' pom.xml | head -2 | tail -1 | cut -d'>' -f2 |cut -d'<' -f1)" >> $GITHUB_ENV

.github/workflows/linter.yml

@@ -32,7 +32,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Check Out
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install
         run: |
           python -m pip install --upgrade pip

.github/workflows/main-sonar-check.yml

@@ -32,7 +32,7 @@ jobs:
     name: Main Sonar JaCoCo Build
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

.github/workflows/rat.yml

@@ -27,7 +27,7 @@ jobs:
   build:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:

.github/workflows/sonar-check.yml

@@ -29,11 +29,11 @@ concurrency:
 
 jobs:
   build:
-    if: github.repository == 'apache/cloudstack'
+    if: github.repository == 'apache/cloudstack' && github.event.pull_request.head.repo.full_name == github.repository
     name: Sonar JaCoCo Coverage
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: "refs/pull/${{ github.event.number }}/merge"
           fetch-depth: 0

.github/workflows/ui.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Node
         uses: actions/setup-node@v3
@@ -51,3 +51,12 @@ jobs:
           npm run build
           npm run lint
           npm run test:unit
+
+      - uses: codecov/codecov-action@v3
+        with:
+          working-directory: ui
+          files: ./coverage/lcov.info
+          fail_ci_if_error: true
+          flags: uitests
+          verbose: true
+          name: codecov

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ default_stages: [commit, push]
 default_language_version:
   # force all unspecified Python hooks to run python3
   python: python3
-minimum_pre_commit_version: "2.18.0"
+minimum_pre_commit_version: "2.17.0"
 repos:
   - repo: meta
     hooks:
@@ -31,9 +31,11 @@ repos:
       #- id: check-added-large-files
       - id: check-case-conflict
       #- id: check-executables-have-shebangs
-      #- id: check-merge-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
       - id: check-vcs-permalinks
       #- id: check-yaml
+      - id: destroyed-symlinks
       - id: detect-private-key
         exclude: >
           (?x)
@@ -52,7 +54,7 @@ repos:
         exclude: \.vhd$
       #- id: fix-byte-order-marker
       - id: mixed-line-ending
-        files: \.(java|md|py|txt|yaml|yml)$
+        exclude: \.(cs|xml)$
       # - id: trailing-whitespace
   - repo: https://github.com/pycqa/flake8
     rev: 6.1.0

PULL_REQUEST_TEMPLATE.md

@@ -10,10 +10,10 @@ This PR...
 <!-- For addressing multiple issues/PRs, use multiple "Fixes: #<id>" -->
 <!-- Fixes: # -->
 
-<!--- ********************************************************************************* -->
-<!--- NOTE: AUTOMATATION USES THE DESCRIPTIONS TO SET LABELS AND PRODUCE DOCUMENTATION. -->
+<!--- ******************************************************************************* -->
+<!--- NOTE: AUTOMATION USES THE DESCRIPTIONS TO SET LABELS AND PRODUCE DOCUMENTATION. -->
 <!--- PLEASE PUT AN 'X' in only **ONE** box -->
-<!--- ********************************************************************************* -->
+<!--- ******************************************************************************* -->
 
 ### Types of changes
 
@@ -22,6 +22,7 @@ This PR...
 - [ ] Bug fix (non-breaking change which fixes an issue)
 - [ ] Enhancement (improves an existing feature and functionality)
 - [ ] Cleanup (Code refactoring and cleanup, that may add test cases)
+- [ ] build/CI
 
 ### Feature/Enhancement Scale or Bug Severity
 
@@ -43,8 +44,12 @@ This PR...
 
 
 ### How Has This Been Tested?
+
 <!-- Please describe in detail how you tested your changes. -->
 <!-- Include details of your testing environment, and the tests you ran to -->
+
+#### How did you try to break this feature and the system with this change?
+
 <!-- see how your change affects other areas of the code, etc. -->
 
 

README.md

@@ -83,6 +83,13 @@ contribution. Note that you do not have to be a developer in order to contribute
 to Apache CloudStack. We need folks to help with documentation, translation,
 promotion etc. See our contribution [page](http://cloudstack.apache.org/contribute.html).
 
+If you are a frequent contributors, you can request to be added as collaborators
+(see https://cwiki.apache.org/confluence/display/INFRA/Git+-+.asf.yaml+features#Git.asf.yamlfeatures-AssigningexternalcollaboratorswiththetriageroleonGitHub)
+to our GitHub repos. This allows you to use project GitHub with ability to report
+issue with tags, and be assigned to issues and PRs. This is done via the .asf.yaml
+file in this repo.
+You may do so by sharing your GitHub users ID or raise a GitHub issue.
+
 If you're interested in learning more or participating in the Apache CloudStack
 project, the mailing lists are the best way to do that. While the project has
 several communications channels, the [mailing lists](http://cloudstack.apache.org/mailing-lists.html) are the most active and the

agent/conf/agent.properties

@@ -279,6 +279,11 @@ hypervisor.type=kvm
 # If this parameter is used, property host.overcommit.mem.mb must be set to 0.
 #host.reserved.mem.mb=1024
 
+# Number of CPU cores to subtract from advertised available cores.
+# These are reserved for system activity, or otherwise share host CPU resources with
+# CloudStack VM allocation.
+# host.reserved.cpu.count = 0
+
 # The model of Watchdog timer to present to the Guest.
 # For all models refer to the libvirt documentation.
 #vm.watchdog.model=i6300esb
@@ -402,3 +407,15 @@ iscsi.session.cleanup.enabled=false
 # The path of an executable file/script for host health check for CloudStack to Auto Disable/Enable the host
 # depending on the return value of the file/script
 # agent.health.check.script.path=
+
+# Time interval (in milliseconds) between KVM heartbeats.
+# kvm.heartbeat.update.frequency=60000
+
+# Number of maximum tries to KVM heartbeats.
+# kvm.heartbeat.update.max.tries=5
+
+# Time amount (in milliseconds) for the KVM heartbeat retry sleep.
+# kvm.heartbeat.update.retry.sleep=10000
+
+# Timeout (in milliseconds) of the KVM heartbeat checker.
+# kvm.heartbeat.checker.timeout=360000

agent/src/main/java/com/cloud/agent/Agent.java

@@ -40,6 +40,9 @@
 
 import javax.naming.ConfigurationException;
 
+import com.cloud.resource.AgentStatusUpdater;
+import com.cloud.resource.ResourceStatusUpdater;
+import com.cloud.agent.api.PingAnswer;
 import com.cloud.utils.NumbersUtil;
 import org.apache.cloudstack.agent.lb.SetupMSListAnswer;
 import org.apache.cloudstack.agent.lb.SetupMSListCommand;
@@ -101,7 +104,7 @@
  *         For more configuration options, see the individual types.
  *
  **/
-public class Agent implements HandlerFactory, IAgentControl {
+public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater {
     protected Logger logger = LogManager.getLogger(getClass());
 
     public enum ExitStatus {
@@ -410,6 +413,20 @@ public void scheduleWatch(final Link link, final Request request, final long del
         }
     }
 
+    public void triggerUpdate() {
+        PingCommand command = _resource.getCurrentStatus(getId());
+        command.setOutOfBand(true);
+        logger.debug("Sending out of band ping");
+
+        final Request request = new Request(_id, -1, command, false);
+        request.setSequence(getNextSequence());
+        try {
+            _link.send(request.toBytes());
+        } catch (final ClosedChannelException e) {
+            logger.warn("Unable to send ping update: " + request.toString());
+        }
+    }
+
     protected void cancelTasks() {
         synchronized (_watchList) {
             for (final WatchTask task : _watchList) {
@@ -462,6 +479,10 @@ public void sendStartup(final Link link) {
             } catch (final ClosedChannelException e) {
                 logger.warn("Unable to send request: " + request.toString());
             }
+
+            if (_resource instanceof ResourceStatusUpdater) {
+                ((ResourceStatusUpdater) _resource).registerStatusUpdater(this);
+            }
         }
     }
 
@@ -823,6 +844,9 @@ public void processResponse(final Response response, final Link link) {
                     listener.processControlResponse(response, (AgentControlAnswer)answer);
                 }
             }
+        } else if (answer instanceof PingAnswer && (((PingAnswer) answer).isSendStartup()) && _reconnectAllowed) {
+            logger.info("Management server requested startup command to reinitialize the agent");
+            sendStartup(link);
         } else {
             setLastPingResponseTime();
         }

agent/src/main/java/com/cloud/agent/dao/impl/PropertiesStorage.java

@@ -93,11 +93,14 @@ public synchronized boolean configure(String name, Map<String, Object> params) {
             file = new File(path);
             try {
                 if (!file.createNewFile()) {
-                    logger.error("Unable to create _file: " + file.getAbsolutePath());
+                    logger.error(String.format("Unable to create _file: %s", file.getAbsolutePath()));
                     return false;
                 }
             } catch (IOException e) {
-                logger.error("Unable to create _file: " + file.getAbsolutePath(), e);
+                logger.error(String.format("Unable to create file: %s", file.getAbsolutePath()));
+                if (logger.isDebugEnabled()) {
+                    logger.debug(String.format("IOException while trying to create file: %s", file.getAbsolutePath()), e);
+                }
                 return false;
             }
         }