[VC-43753] CyberArk(client): extract cluster UID from data readings

.envrc.template

@@ -0,0 +1,21 @@
+# Example .envrc file for use with direnv.
+# Copy this file to .envrc and edit the values as required.
+# Do not check in your .envrc file to source control as it may contain secrets.
+
+# The following variables are required by the E2E test script: ./hack/e2e/test.sh.
+export VEN_API_KEY=       # your Venafi Cloud API key with full permissions
+export VEN_API_KEY_PULL=  # your Venafi Cloud API key with pull-only permissions
+export VEN_ZONE=          # the Venafi Cloud zone to use for certificate requests
+export VEN_VCP_REGION=    # the Venafi Cloud region to use (us or eu)
+export VEN_API_HOST=      # the Venafi Cloud API host (usually api.venafi.cloud or api.venafi.eu)
+export OCI_BASE=          # the base URL for the OCI registry where the Agent chart and image will be pushed
+export CLOUDSDK_CORE_PROJECT= # the GCP project ID where a GKE cluster will be created.
+export CLOUDSDK_COMPUTE_ZONE= # the GCP zone where a GKE cluster will be created. E.g. europe-west2-b
+export CLUSTER_NAME=          # the name of the GKE cluster which will be created. E.g. cluster-1
+
+# The following variables are required for CyberArk / MachineHub integration tests.
+export ARK_SUBDOMAIN=      # your CyberArk tenant subdomain
+export ARK_USERNAME=       # your CyberArk username
+export ARK_SECRET=         # your CyberArk password
+# OPTIONAL: the URL for the CyberArk Discovery API if not using the production environment
+export ARK_DISCOVERY_API=https://platform-discovery.integration-cyberark.cloud/api/v2

.gitignore

@@ -14,3 +14,4 @@ predicate.json
 *.tgz
 
 _bin
+.envrc

api/datareading.go

@@ -1,8 +1,12 @@
 package api
 
 import (
+	"bytes"
 	"encoding/json"
 	"time"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/version"
 )
 
 // DataReadingsPost is the payload in the upload request.
@@ -28,8 +32,8 @@ type DataReading struct {
 type GatheredResource struct {
 	// Resource is a reference to a k8s object that was found by the informer
 	// should be of type unstructured.Unstructured, raw Object
-	Resource  interface{}
-	DeletedAt Time
+	Resource  interface{} `json:"resource"`
+	DeletedAt Time        `json:"deleted_at,omitempty"`
 }
 
 func (v GatheredResource) MarshalJSON() ([]byte, error) {
@@ -48,3 +52,32 @@ func (v GatheredResource) MarshalJSON() ([]byte, error) {
 
 	return json.Marshal(data)
 }
+
+func (v *GatheredResource) UnmarshalJSON(data []byte) error {
+	var tmpResource struct {
+		Resource  *unstructured.Unstructured `json:"resource"`
+		DeletedAt Time                       `json:"deleted_at,omitempty"`
+	}
+
+	d := json.NewDecoder(bytes.NewReader(data))
+	d.DisallowUnknownFields()
+
+	if err := d.Decode(&tmpResource); err != nil {
+		return err
+	}
+	v.Resource = tmpResource.Resource
+	v.DeletedAt = tmpResource.DeletedAt
+	return nil
+}
+
+// DynamicData is the DataReading.Data returned by the k8s.DataGathererDynamic
+// gatherer
+type DynamicData struct {
+	Items []*GatheredResource `json:"items"`
+}
+
+// DiscoveryData is the DataReading.Data returned by the k8s.ConfigDiscovery
+// gatherer
+type DiscoveryData struct {
+	ServerVersion *version.Info `json:"server_version"`
+}

examples/machinehub.yaml

@@ -0,0 +1,12 @@
+# An example agent config for MachineHub output mode.
+#
+# For example:
+#
+#  export ARK_SUBDOMAIN=      # your CyberArk tenant subdomain
+#  export ARK_USERNAME=       # your CyberArk username
+#  export ARK_SECRET=         # your CyberArk password
+#  go run . agent --one-shot --machine-hub -v 6 --agent-config-file ./examples/machinehub.yaml
+
+data-gatherers:
+  - kind: "dummy"
+    name: "dummy"

pkg/agent/config.go

@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"crypto/x509"
 	"fmt"
 	"io"
 	"net/url"
@@ -10,9 +11,11 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/hashicorp/go-multierror"
+	"github.com/jetstack/venafi-connection-lib/http_client"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v3"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/transport"
 
 	"github.com/jetstack/preflight/api"
 	"github.com/jetstack/preflight/pkg/client"
@@ -334,6 +337,7 @@ const (
 	VenafiCloudKeypair          OutputMode = "Venafi Cloud Key Pair Service Account"
 	VenafiCloudVenafiConnection OutputMode = "Venafi Cloud VenafiConnection"
 	LocalFile                   OutputMode = "Local File"
+	MachineHub                  OutputMode = "MachineHub"
 )
 
 // The command-line flags and the config file are combined into this struct by
@@ -420,6 +424,9 @@ func ValidateAndCombineConfig(log logr.Logger, cfg Config, flags AgentCmdFlags)
 		case !flags.VenafiCloudMode && flags.CredentialsPath != "":
 			mode = JetstackSecureOAuth
 			reason = "--credentials-file was specified without --venafi-cloud"
+		case flags.MachineHubMode:
+			mode = MachineHub
+			reason = "--machine-hub was specified"
 		case flags.OutputPath != "":
 			mode = LocalFile
 			reason = "--output-path was specified"
@@ -433,6 +440,7 @@ func ValidateAndCombineConfig(log logr.Logger, cfg Config, flags AgentCmdFlags)
 				" - Use --venafi-connection for the " + string(VenafiCloudVenafiConnection) + " mode.\n" +
 				" - Use --credentials-file alone if you want to use the " + string(JetstackSecureOAuth) + " mode.\n" +
 				" - Use --api-token if you want to use the " + string(JetstackSecureAPIToken) + " mode.\n" +
+				" - Use --machine-hub if you want to use the " + string(MachineHub) + " mode.\n" +
 				" - Use --output-path or output-path in the config file for " + string(LocalFile) + " mode.")
 		}
 
@@ -548,6 +556,13 @@ func ValidateAndCombineConfig(log logr.Logger, cfg Config, flags AgentCmdFlags)
 			}
 			organizationID = cfg.OrganizationID
 			clusterID = cfg.ClusterID
+		case MachineHub:
+			if cfg.ClusterID != "" {
+				log.Info(fmt.Sprintf(`Ignoring the cluster_id field in the config file. This field is not needed in %s mode.`, res.OutputMode))
+			}
+			if cfg.OrganizationID != "" {
+				log.Info(fmt.Sprintf(`Ignoring the organization_id field in the config file. This field is not needed in %s mode.`, res.OutputMode))
+			}
 		}
 		res.OrganizationID = organizationID
 		res.ClusterID = clusterID
@@ -762,6 +777,17 @@ func validateCredsAndCreateClient(log logr.Logger, flagCredentialsPath, flagClie
 		}
 	case LocalFile:
 		outputClient = client.NewFileClient(cfg.OutputPath)
+	case MachineHub:
+		var (
+			err     error
+			rootCAs *x509.CertPool
+		)
+		httpClient := http_client.NewDefaultClient(version.UserAgent(), rootCAs)
+		httpClient.Transport = transport.NewDebuggingRoundTripper(httpClient.Transport, transport.DebugByContext)
+		outputClient, err = client.NewCyberArk(httpClient)
+		if err != nil {
+			errs = multierror.Append(errs, err)
+		}
 	default:
 		panic(fmt.Errorf("programmer mistake: output mode not implemented: %s", cfg.OutputMode))
 	}

pkg/agent/config_test.go

@@ -199,6 +199,7 @@ func Test_ValidateAndCombineConfig(t *testing.T) {
 			 - Use --venafi-connection for the Venafi Cloud VenafiConnection mode.
 			 - Use --credentials-file alone if you want to use the Jetstack Secure OAuth mode.
 			 - Use --api-token if you want to use the Jetstack Secure API Token mode.
+			 - Use --machine-hub if you want to use the MachineHub mode.
 			 - Use --output-path or output-path in the config file for Local File mode.`))
 		assert.Nil(t, cl)
 	})
@@ -617,6 +618,38 @@ func Test_ValidateAndCombineConfig(t *testing.T) {
 		assert.Equal(t, VenafiCloudVenafiConnection, got.OutputMode)
 	})
 
+	t.Run("--machine-hub selects MachineHub mode", func(t *testing.T) {
+		t.Setenv("POD_NAMESPACE", "venafi")
+		t.Setenv("KUBECONFIG", withFile(t, fakeKubeconfig))
+		t.Setenv("ARK_SUBDOMAIN", "tlspk")
+		t.Setenv("ARK_USERNAME", "first_last@cyberark.cloud.123456")
+		t.Setenv("ARK_SECRET", "test-secret")
+		got, cl, err := ValidateAndCombineConfig(discardLogs(),
+			withConfig(""),
+			withCmdLineFlags("--period", "1m", "--machine-hub"))
+		require.NoError(t, err)
+		assert.Equal(t, MachineHub, got.OutputMode)
+		assert.IsType(t, &client.CyberArkClient{}, cl)
+	})
+
+	t.Run("--machine-hub without required environment variables", func(t *testing.T) {
+		t.Setenv("POD_NAMESPACE", "venafi")
+		t.Setenv("KUBECONFIG", withFile(t, fakeKubeconfig))
+		t.Setenv("ARK_SUBDOMAIN", "")
+		t.Setenv("ARK_USERNAME", "")
+		t.Setenv("ARK_SECRET", "")
+		got, cl, err := ValidateAndCombineConfig(discardLogs(),
+			withConfig(""),
+			withCmdLineFlags("--period", "1m", "--machine-hub"))
+		assert.Equal(t, CombinedConfig{}, got)
+		assert.Nil(t, cl)
+		assert.EqualError(t, err, testutil.Undent(`
+			validating creds: failed loading config using the MachineHub mode: 1 error occurred:
+				* missing environment variables: ARK_SUBDOMAIN, ARK_USERNAME, ARK_SECRET
+
+	   `))
+	})
+
 	t.Run("argument: --output-file selects local file mode", func(t *testing.T) {
 		log, gotLog := recordLogs(t)
 		got, outputClient, err := ValidateAndCombineConfig(log,

pkg/agent/run.go

@@ -305,6 +305,14 @@ func gatherAndOutputData(ctx context.Context, eventf Eventf, config CombinedConf
 	var readings []*api.DataReading
 
 	if config.InputPath != "" {
+		// TODO(wallrj): The datareadings read from disk can not yet be pushed
+		// to the CyberArk Discovery and Context API. Why? Because they have
+		// simple data types such as map[string]interface{}. In contrast, the
+		// data from data gatherers can be cast to rich types like DynamicData
+		// or DiscoveryData The CyberArk dataupload client requires the data to
+		// have rich types to convert it to the Discovery and Context snapshots
+		// format. Consider refactoring testutil.ParseDataReadings so that it
+		// can be used here.
 		log.V(logs.Debug).Info("Reading data from local file", "inputPath", config.InputPath)
 		data, err := os.ReadFile(config.InputPath)
 		if err != nil {