diff --git a/.envrc.template b/.envrc.template new file mode 100644 index 00000000..6765a4d2 --- /dev/null +++ b/.envrc.template @@ -0,0 +1,21 @@ +# Example .envrc file for use with direnv. +# Copy this file to .envrc and edit the values as required. +# Do not check in your .envrc file to source control as it may contain secrets. + +# The following variables are required by the E2E test script: ./hack/e2e/test.sh. +export VEN_API_KEY= # your Venafi Cloud API key with full permissions +export VEN_API_KEY_PULL= # your Venafi Cloud API key with pull-only permissions +export VEN_ZONE= # the Venafi Cloud zone to use for certificate requests +export VEN_VCP_REGION= # the Venafi Cloud region to use (us or eu) +export VEN_API_HOST= # the Venafi Cloud API host (usually api.venafi.cloud or api.venafi.eu) +export OCI_BASE= # the base URL for the OCI registry where the Agent chart and image will be pushed +export CLOUDSDK_CORE_PROJECT= # the GCP project ID where a GKE cluster will be created. +export CLOUDSDK_COMPUTE_ZONE= # the GCP zone where a GKE cluster will be created. E.g. europe-west2-b +export CLUSTER_NAME= # the name of the GKE cluster which will be created. E.g. cluster-1 + +# The following variables are required for CyberArk / MachineHub integration tests. +export ARK_SUBDOMAIN= # your CyberArk tenant subdomain +export ARK_USERNAME= # your CyberArk username +export ARK_SECRET= # your CyberArk password +# OPTIONAL: the URL for the CyberArk Discovery API if not using the production environment +export ARK_DISCOVERY_API=https://platform-discovery.integration-cyberark.cloud/api/v2 diff --git a/.gitignore b/.gitignore index 17966351..ec041a7c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ predicate.json *.tgz _bin +.envrc diff --git a/api/datareading.go b/api/datareading.go index 54637f3c..75e7dee3 100644 --- a/api/datareading.go +++ b/api/datareading.go @@ -1,8 +1,12 @@ package api import ( + "bytes" "encoding/json" "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/version" ) // DataReadingsPost is the payload in the upload request. @@ -28,8 +32,8 @@ type DataReading struct { type GatheredResource struct { // Resource is a reference to a k8s object that was found by the informer // should be of type unstructured.Unstructured, raw Object - Resource interface{} - DeletedAt Time + Resource interface{} `json:"resource"` + DeletedAt Time `json:"deleted_at,omitempty"` } func (v GatheredResource) MarshalJSON() ([]byte, error) { @@ -48,3 +52,32 @@ func (v GatheredResource) MarshalJSON() ([]byte, error) { return json.Marshal(data) } + +func (v *GatheredResource) UnmarshalJSON(data []byte) error { + var tmpResource struct { + Resource *unstructured.Unstructured `json:"resource"` + DeletedAt Time `json:"deleted_at,omitempty"` + } + + d := json.NewDecoder(bytes.NewReader(data)) + d.DisallowUnknownFields() + + if err := d.Decode(&tmpResource); err != nil { + return err + } + v.Resource = tmpResource.Resource + v.DeletedAt = tmpResource.DeletedAt + return nil +} + +// DynamicData is the DataReading.Data returned by the k8s.DataGathererDynamic +// gatherer +type DynamicData struct { + Items []*GatheredResource `json:"items"` +} + +// DiscoveryData is the DataReading.Data returned by the k8s.ConfigDiscovery +// gatherer +type DiscoveryData struct { + ServerVersion *version.Info `json:"server_version"` +} diff --git a/examples/machinehub.yaml b/examples/machinehub.yaml new file mode 100644 index 00000000..94b19d48 --- /dev/null +++ b/examples/machinehub.yaml @@ -0,0 +1,12 @@ +# An example agent config for MachineHub output mode. +# +# For example: +# +# export ARK_SUBDOMAIN= # your CyberArk tenant subdomain +# export ARK_USERNAME= # your CyberArk username +# export ARK_SECRET= # your CyberArk password +# go run . agent --one-shot --machine-hub -v 6 --agent-config-file ./examples/machinehub.yaml + +data-gatherers: + - kind: "dummy" + name: "dummy" diff --git a/pkg/agent/config.go b/pkg/agent/config.go index 2f691938..34967e19 100644 --- a/pkg/agent/config.go +++ b/pkg/agent/config.go @@ -1,6 +1,7 @@ package agent import ( + "crypto/x509" "fmt" "io" "net/url" @@ -10,9 +11,11 @@ import ( "github.com/go-logr/logr" "github.com/hashicorp/go-multierror" + "github.com/jetstack/venafi-connection-lib/http_client" "github.com/spf13/cobra" "gopkg.in/yaml.v3" "k8s.io/client-go/rest" + "k8s.io/client-go/transport" "github.com/jetstack/preflight/api" "github.com/jetstack/preflight/pkg/client" @@ -334,6 +337,7 @@ const ( VenafiCloudKeypair OutputMode = "Venafi Cloud Key Pair Service Account" VenafiCloudVenafiConnection OutputMode = "Venafi Cloud VenafiConnection" LocalFile OutputMode = "Local File" + MachineHub OutputMode = "MachineHub" ) // The command-line flags and the config file are combined into this struct by @@ -420,6 +424,9 @@ func ValidateAndCombineConfig(log logr.Logger, cfg Config, flags AgentCmdFlags) case !flags.VenafiCloudMode && flags.CredentialsPath != "": mode = JetstackSecureOAuth reason = "--credentials-file was specified without --venafi-cloud" + case flags.MachineHubMode: + mode = MachineHub + reason = "--machine-hub was specified" case flags.OutputPath != "": mode = LocalFile reason = "--output-path was specified" @@ -433,6 +440,7 @@ func ValidateAndCombineConfig(log logr.Logger, cfg Config, flags AgentCmdFlags) " - Use --venafi-connection for the " + string(VenafiCloudVenafiConnection) + " mode.\n" + " - Use --credentials-file alone if you want to use the " + string(JetstackSecureOAuth) + " mode.\n" + " - Use --api-token if you want to use the " + string(JetstackSecureAPIToken) + " mode.\n" + + " - Use --machine-hub if you want to use the " + string(MachineHub) + " mode.\n" + " - Use --output-path or output-path in the config file for " + string(LocalFile) + " mode.") } @@ -548,6 +556,13 @@ func ValidateAndCombineConfig(log logr.Logger, cfg Config, flags AgentCmdFlags) } organizationID = cfg.OrganizationID clusterID = cfg.ClusterID + case MachineHub: + if cfg.ClusterID != "" { + log.Info(fmt.Sprintf(`Ignoring the cluster_id field in the config file. This field is not needed in %s mode.`, res.OutputMode)) + } + if cfg.OrganizationID != "" { + log.Info(fmt.Sprintf(`Ignoring the organization_id field in the config file. This field is not needed in %s mode.`, res.OutputMode)) + } } res.OrganizationID = organizationID res.ClusterID = clusterID @@ -762,6 +777,17 @@ func validateCredsAndCreateClient(log logr.Logger, flagCredentialsPath, flagClie } case LocalFile: outputClient = client.NewFileClient(cfg.OutputPath) + case MachineHub: + var ( + err error + rootCAs *x509.CertPool + ) + httpClient := http_client.NewDefaultClient(version.UserAgent(), rootCAs) + httpClient.Transport = transport.NewDebuggingRoundTripper(httpClient.Transport, transport.DebugByContext) + outputClient, err = client.NewCyberArk(httpClient) + if err != nil { + errs = multierror.Append(errs, err) + } default: panic(fmt.Errorf("programmer mistake: output mode not implemented: %s", cfg.OutputMode)) } diff --git a/pkg/agent/config_test.go b/pkg/agent/config_test.go index fe5211d3..70951b57 100644 --- a/pkg/agent/config_test.go +++ b/pkg/agent/config_test.go @@ -199,6 +199,7 @@ func Test_ValidateAndCombineConfig(t *testing.T) { - Use --venafi-connection for the Venafi Cloud VenafiConnection mode. - Use --credentials-file alone if you want to use the Jetstack Secure OAuth mode. - Use --api-token if you want to use the Jetstack Secure API Token mode. + - Use --machine-hub if you want to use the MachineHub mode. - Use --output-path or output-path in the config file for Local File mode.`)) assert.Nil(t, cl) }) @@ -617,6 +618,38 @@ func Test_ValidateAndCombineConfig(t *testing.T) { assert.Equal(t, VenafiCloudVenafiConnection, got.OutputMode) }) + t.Run("--machine-hub selects MachineHub mode", func(t *testing.T) { + t.Setenv("POD_NAMESPACE", "venafi") + t.Setenv("KUBECONFIG", withFile(t, fakeKubeconfig)) + t.Setenv("ARK_SUBDOMAIN", "tlspk") + t.Setenv("ARK_USERNAME", "first_last@cyberark.cloud.123456") + t.Setenv("ARK_SECRET", "test-secret") + got, cl, err := ValidateAndCombineConfig(discardLogs(), + withConfig(""), + withCmdLineFlags("--period", "1m", "--machine-hub")) + require.NoError(t, err) + assert.Equal(t, MachineHub, got.OutputMode) + assert.IsType(t, &client.CyberArkClient{}, cl) + }) + + t.Run("--machine-hub without required environment variables", func(t *testing.T) { + t.Setenv("POD_NAMESPACE", "venafi") + t.Setenv("KUBECONFIG", withFile(t, fakeKubeconfig)) + t.Setenv("ARK_SUBDOMAIN", "") + t.Setenv("ARK_USERNAME", "") + t.Setenv("ARK_SECRET", "") + got, cl, err := ValidateAndCombineConfig(discardLogs(), + withConfig(""), + withCmdLineFlags("--period", "1m", "--machine-hub")) + assert.Equal(t, CombinedConfig{}, got) + assert.Nil(t, cl) + assert.EqualError(t, err, testutil.Undent(` + validating creds: failed loading config using the MachineHub mode: 1 error occurred: + * missing environment variables: ARK_SUBDOMAIN, ARK_USERNAME, ARK_SECRET + + `)) + }) + t.Run("argument: --output-file selects local file mode", func(t *testing.T) { log, gotLog := recordLogs(t) got, outputClient, err := ValidateAndCombineConfig(log, diff --git a/pkg/agent/run.go b/pkg/agent/run.go index 7bf737c3..19883d30 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -305,6 +305,14 @@ func gatherAndOutputData(ctx context.Context, eventf Eventf, config CombinedConf var readings []*api.DataReading if config.InputPath != "" { + // TODO(wallrj): The datareadings read from disk can not yet be pushed + // to the CyberArk Discovery and Context API. Why? Because they have + // simple data types such as map[string]interface{}. In contrast, the + // data from data gatherers can be cast to rich types like DynamicData + // or DiscoveryData The CyberArk dataupload client requires the data to + // have rich types to convert it to the Discovery and Context snapshots + // format. Consider refactoring testutil.ParseDataReadings so that it + // can be used here. log.V(logs.Debug).Info("Reading data from local file", "inputPath", config.InputPath) data, err := os.ReadFile(config.InputPath) if err != nil { diff --git a/pkg/client/client_cyberark.go b/pkg/client/client_cyberark.go index c5800076..d27d148b 100644 --- a/pkg/client/client_cyberark.go +++ b/pkg/client/client_cyberark.go @@ -1,9 +1,189 @@ package client import ( + "context" + "errors" + "fmt" + "net/http" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/jetstack/preflight/api" + "github.com/jetstack/preflight/pkg/internal/cyberark" "github.com/jetstack/preflight/pkg/internal/cyberark/dataupload" + "github.com/jetstack/preflight/pkg/version" ) -type CyberArkClient = dataupload.CyberArkClient +// CyberArkClient is a client for publishing data readings to CyberArk's discoverycontext API. +type CyberArkClient struct { + configLoader cyberark.ClientConfigLoader + httpClient *http.Client +} + +var _ Client = &CyberArkClient{} + +// NewCyberArk initializes a CyberArk client using configuration from environment variables. +// It requires an HTTP client to be provided, which will be used for making requests. +// The environment variables ARK_SUBDOMAIN, ARK_USERNAME, and ARK_SECRET must be set for authentication. +// If the configuration is invalid or missing, an error is returned. +func NewCyberArk(httpClient *http.Client) (*CyberArkClient, error) { + configLoader := cyberark.LoadClientConfigFromEnvironment + _, err := configLoader() + if err != nil { + return nil, err + } + return &CyberArkClient{ + configLoader: configLoader, + httpClient: httpClient, + }, nil +} + +// PostDataReadingsWithOptions uploads data readings to CyberArk. +// It initializes a data upload client with the configured HTTP client and credentials, +// then uploads a snapshot. +// The supplied Options are not used by this publisher. +func (o *CyberArkClient) PostDataReadingsWithOptions(ctx context.Context, readings []*api.DataReading, _ Options) error { + cfg, err := o.configLoader() + if err != nil { + return err + } + datauploadClient, err := cyberark.NewDatauploadClient(ctx, o.httpClient, cfg) + if err != nil { + return fmt.Errorf("while initializing data upload client: %s", err) + } + + snapshot, err := ConvertDataReadingsToCyberarkSnapshot(readings) + if err != nil { + return fmt.Errorf("while converting data readings: %s", err) + } + + err = datauploadClient.PutSnapshot(ctx, snapshot) + if err != nil { + return fmt.Errorf("while uploading snapshot: %s", err) + } + return nil +} + +type resourceData map[string][]*unstructured.Unstructured + +// The names of Datagatherers which have the data to populate the Cyberark +// Snapshot mapped to the key in the Cyberark snapshot. +var gathererNameToResourceDataKeyMap = map[string]string{ + "ark/secrets": "secrets", + "ark/serviceaccounts": "serviceaccounts", + "ark/roles": "roles", + "ark/clusterroles": "clusterroles", + "ark/rolebindings": "rolebindings", + "ark/clusterrolebindings": "clusterrolebindings", + "ark/jobs": "jobs", + "ark/cronjobs": "cronjobs", + "ark/deployments": "deployments", + "ark/statefulsets": "statefulsets", + "ark/daemonsets": "daemonsets", + "ark/pods": "pods", +} + +// extractClusterUIDFromReading converts the opaque data from a DynamicData +// reading to Unstructured Namespace resources, and finds the UID of the +// `kube-system` namespace. +// This UID can be used as a unique identifier for the Kubernetes cluster. +// - https://venafi.slack.com/archives/C04SQR5DAD7/p1747825325264979 +// - https://github.com/kubernetes/kubernetes/issues/77487#issuecomment-489786023 +func extractClusterUIDFromReading(reading *api.DataReading) (string, error) { + resources, err := extractResourceListFromReading(reading) + if err != nil { + return "", err + } + for _, resource := range resources { + if resource.GetName() == "kube-system" { + return string(resource.GetUID()), nil + } + } + return "", fmt.Errorf("kube-system namespace UID not found in data reading: %v", reading) +} + +// extractServerVersionFromReading converts the opaque data from a DiscoveryData +// data reding to allow access to the Kubernetes version fields within. +func extractServerVersionFromReading(reading *api.DataReading) (string, error) { + data, ok := reading.Data.(*api.DiscoveryData) + if !ok { + return "", fmt.Errorf("failed to convert data: %s", reading.DataGatherer) + } + if data.ServerVersion == nil { + return "unknown", nil + } + return data.ServerVersion.GitVersion, nil +} + +// extractResourceListFromReading converts the opaque data from a DynamicData +// data reading to Unstructured resources, to allow access to the metadata and +// other kubernetes API fields. +func extractResourceListFromReading(reading *api.DataReading) ([]*unstructured.Unstructured, error) { + data, ok := reading.Data.(*api.DynamicData) + if !ok { + return nil, fmt.Errorf("failed to convert data: %s", reading.DataGatherer) + } + items := data.Items + resources := make([]*unstructured.Unstructured, len(items)) + for i, item := range items { + if resource, ok := item.Resource.(*unstructured.Unstructured); ok { + resources[i] = resource + } else { + return nil, fmt.Errorf("failed to convert resource: %#v", item) + } + } + return resources, nil +} -var NewCyberArkClient = dataupload.New +// ConvertDataReadingsToCyberarkSnapshot converts DataReadings to the Cyberark +// Snapshot format. +func ConvertDataReadingsToCyberarkSnapshot( + readings []*api.DataReading, +) (s dataupload.Snapshot, _ error) { + k8sVersion := "" + clusterID := "" + resourceData := resourceData{} + for _, reading := range readings { + if reading.DataGatherer == "ark/discovery" { + var err error + k8sVersion, err = extractServerVersionFromReading(reading) + if err != nil { + return s, fmt.Errorf("while extracting server version from data-reading: %s", err) + } + } + if reading.DataGatherer == "ark/namespaces" { + var err error + clusterID, err = extractClusterUIDFromReading(reading) + if err != nil { + return s, fmt.Errorf("while extracting cluster UID from data-reading: %s", err) + } + } + if key, found := gathererNameToResourceDataKeyMap[reading.DataGatherer]; found { + resources, err := extractResourceListFromReading(reading) + if err != nil { + return s, fmt.Errorf("while extracting resource list from data-reading: %s", err) + } + resourceData[key] = append(resourceData[key], resources...) + } + } + if clusterID == "" { + return s, errors.New("failed to compute a clusterID from the data-readings") + } + return dataupload.Snapshot{ + AgentVersion: version.PreflightVersion, + K8SVersion: k8sVersion, + ClusterID: clusterID, + Secrets: resourceData["secrets"], + ServiceAccounts: resourceData["serviceaccounts"], + Roles: resourceData["roles"], + ClusterRoles: resourceData["clusterroles"], + RoleBindings: resourceData["rolebindings"], + ClusterRoleBindings: resourceData["clusterrolebindings"], + Jobs: resourceData["jobs"], + CronJobs: resourceData["cronjobs"], + Deployments: resourceData["deployments"], + Statefulsets: resourceData["statefulsets"], + Daemonsets: resourceData["daemonsets"], + Pods: resourceData["pods"], + }, nil +} diff --git a/pkg/client/client_cyberark_test.go b/pkg/client/client_cyberark_test.go new file mode 100644 index 00000000..ec79c567 --- /dev/null +++ b/pkg/client/client_cyberark_test.go @@ -0,0 +1,132 @@ +package client_test + +import ( + "crypto/x509" + "encoding/json" + "errors" + "os" + "testing" + + "github.com/jetstack/venafi-connection-lib/http_client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/transport" + "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" + + "github.com/jetstack/preflight/api" + "github.com/jetstack/preflight/pkg/client" + "github.com/jetstack/preflight/pkg/internal/cyberark" + "github.com/jetstack/preflight/pkg/internal/cyberark/servicediscovery" + "github.com/jetstack/preflight/pkg/testutil" + "github.com/jetstack/preflight/pkg/version" + + _ "k8s.io/klog/v2/ktesting/init" +) + +func genNamespace(name string) *unstructured.Unstructured { + o := &unstructured.Unstructured{} + o.SetAPIVersion("") + o.SetKind("Namespace") + o.SetName(name) + return o +} + +func genArkNamespacesDataReading(clusterID types.UID) *api.DataReading { + kubeSystemNamespace := genNamespace("kube-system") + kubeSystemNamespace.SetUID(clusterID) + return &api.DataReading{ + ClusterID: "ignored-tlspk-cluster-id", + DataGatherer: "ark/namespaces", + Data: &api.DynamicData{ + Items: []*api.GatheredResource{ + { + Resource: kubeSystemNamespace, + }, + { + Resource: genNamespace("kube-public"), + }, + { + Resource: genNamespace("venafi"), + }, + { + Resource: genNamespace("cert-manager"), + }, + }, + }, + SchemaVersion: "v1", + } +} + +// TestCyberArkClient_PostDataReadingsWithOptions_MockAPI demonstrates that the +// dataupload code works with the mock CyberArk APIs. +// The environment variables are chosen to match those expected by the mock +// server. +func TestCyberArkClient_PostDataReadingsWithOptions_MockAPI(t *testing.T) { + t.Setenv("ARK_SUBDOMAIN", servicediscovery.MockDiscoverySubdomain) + t.Setenv("ARK_USERNAME", "test@example.com") + t.Setenv("ARK_SECRET", "somepassword") + t.Run("success", func(t *testing.T) { + logger := ktesting.NewLogger(t, ktesting.DefaultConfig) + ctx := klog.NewContext(t.Context(), logger) + + httpClient := testutil.FakeCyberArk(t) + + c, err := client.NewCyberArk(httpClient) + require.NoError(t, err) + + readings := []*api.DataReading{ + genArkNamespacesDataReading("success-cluster-id"), + } + err = c.PostDataReadingsWithOptions(ctx, readings, client.Options{}) + require.NoError(t, err) + }) +} + +// TestCyberArkClient_PostDataReadingsWithOptions_RealAPI demonstrates that the +// dataupload code works with the real CyberArk APIs. +// +// To enable verbose request logging: +// +// go test ./pkg/internal/cyberark/dataupload/... \ +// -v -count 1 -run TestCyberArkClient_PostDataReadingsWithOptions_RealAPI -args -testing.v 6 +func TestCyberArkClient_PostDataReadingsWithOptions_RealAPI(t *testing.T) { + t.Run("success", func(t *testing.T) { + logger := ktesting.NewLogger(t, ktesting.DefaultConfig) + ctx := klog.NewContext(t.Context(), logger) + + var rootCAs *x509.CertPool + httpClient := http_client.NewDefaultClient(version.UserAgent(), rootCAs) + httpClient.Transport = transport.NewDebuggingRoundTripper(httpClient.Transport, transport.DebugByContext) + + c, err := client.NewCyberArk(httpClient) + if err != nil { + if errors.Is(err, cyberark.ErrMissingEnvironmentVariables) { + t.Skipf("Skipping: %s", err) + } + require.NoError(t, err) + } + readings := testutil.ParseDataReadings(t, testutil.ReadGZIP(t, "testdata/example-1/datareadings.json.gz")) + err = c.PostDataReadingsWithOptions(ctx, readings, client.Options{}) + require.NoError(t, err) + }) +} + +func TestConvertDataReadingsToCyberarkSnapshot(t *testing.T) { + dataReadings := testutil.ParseDataReadings(t, testutil.ReadGZIP(t, "testdata/example-1/datareadings.json.gz")) + snapshot, err := client.ConvertDataReadingsToCyberarkSnapshot(dataReadings) + require.NoError(t, err) + + actualSnapshotBytes, err := json.MarshalIndent(snapshot, "", " ") + require.NoError(t, err) + + goldenFilePath := "testdata/example-1/snapshot.json.gz" + if _, update := os.LookupEnv("UPDATE_GOLDEN_FILES"); update { + testutil.WriteGZIP(t, goldenFilePath, actualSnapshotBytes) + } else { + expectedSnapshotBytes := testutil.ReadGZIP(t, goldenFilePath) + assert.JSONEq(t, string(expectedSnapshotBytes), string(actualSnapshotBytes)) + } +} diff --git a/pkg/client/testdata/example-1/README.md b/pkg/client/testdata/example-1/README.md new file mode 100644 index 00000000..3b3d0dd6 --- /dev/null +++ b/pkg/client/testdata/example-1/README.md @@ -0,0 +1,25 @@ +# README + +Data captured from a cert-manager E2E test cluster. + +```bash +cd cert-manager +make e2e-setup +``` + +```bash +cd jetstack-secure +go run . agent \ + --log-level 6 \ + --one-shot \ + --agent-config-file pkg/client/testdata/example-1/agent.yaml \ + --output-path pkg/client/testdata/example-1/datareadings.json +gzip pkg/client/testdata/example-1/datareadings.json +``` + + +To recreate the golden output file: + +```bash +UPDATE_GOLDEN_FILES=true go test ./pkg/client/... -run TestConvertDataReadingsToCyberarkSnapshot +``` diff --git a/pkg/client/testdata/example-1/agent.yaml b/pkg/client/testdata/example-1/agent.yaml new file mode 100644 index 00000000..02df0392 --- /dev/null +++ b/pkg/client/testdata/example-1/agent.yaml @@ -0,0 +1,101 @@ +cluster_id: example-cluster-id +organization_id: example-organization-id +data-gatherers: +# gather k8s apiserver version information +- kind: k8s-discovery + name: ark/discovery +- kind: k8s-dynamic + name: ark/namespaces + config: + resource-type: + version: v1 + resource: namespaces +- kind: k8s-dynamic + name: ark/secrets + config: + resource-type: + version: v1 + resource: secrets + field-selectors: + - type!=kubernetes.io/service-account-token + - type!=kubernetes.io/dockercfg + - type!=kubernetes.io/dockerconfigjson + - type!=kubernetes.io/basic-auth + - type!=kubernetes.io/ssh-auth + - type!=bootstrap.kubernetes.io/token + - type!=helm.sh/release.v1 +- kind: k8s-dynamic + name: ark/serviceaccounts + config: + resource-type: + resource: serviceaccounts + version: v1 +- kind: k8s-dynamic + name: ark/roles + config: + resource-type: + version: v1 + group: rbac.authorization.k8s.io + resource: roles +- kind: k8s-dynamic + name: ark/clusterroles + config: + resource-type: + version: v1 + group: rbac.authorization.k8s.io + resource: clusterroles +- kind: k8s-dynamic + name: ark/rolebindings + config: + resource-type: + version: v1 + group: rbac.authorization.k8s.io + resource: rolebindings +- kind: k8s-dynamic + name: ark/clusterrolebindings + config: + resource-type: + version: v1 + group: rbac.authorization.k8s.io + resource: clusterrolebindings +- kind: k8s-dynamic + name: ark/jobs + config: + resource-type: + version: v1 + group: batch + resource: jobs +- kind: k8s-dynamic + name: ark/cronjobs + config: + resource-type: + version: v1 + group: batch + resource: cronjobs +- kind: k8s-dynamic + name: ark/deployments + config: + resource-type: + version: v1 + group: apps + resource: deployments +- kind: k8s-dynamic + name: ark/statefulsets + config: + resource-type: + version: v1 + group: apps + resource: statefulsets +- kind: k8s-dynamic + name: ark/daemonsets + config: + resource-type: + version: v1 + group: apps + resource: daemonsets +- kind: k8s-dynamic + name: ark/pods + config: + resource-type: + version: v1 + resource: pods diff --git a/pkg/client/testdata/example-1/datareadings.json.gz b/pkg/client/testdata/example-1/datareadings.json.gz new file mode 100644 index 00000000..08f144d6 Binary files /dev/null and b/pkg/client/testdata/example-1/datareadings.json.gz differ diff --git a/pkg/client/testdata/example-1/snapshot.json.gz b/pkg/client/testdata/example-1/snapshot.json.gz new file mode 100644 index 00000000..f0ed79ff Binary files /dev/null and b/pkg/client/testdata/example-1/snapshot.json.gz differ diff --git a/pkg/clusteruid/clusteruid.go b/pkg/clusteruid/clusteruid.go deleted file mode 100644 index 2a5327f2..00000000 --- a/pkg/clusteruid/clusteruid.go +++ /dev/null @@ -1,45 +0,0 @@ -package clusteruid - -import ( - "context" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" -) - -// clusterUIDKey is the context key for storing the cluster UID -type clusterUIDKey struct{} - -// GetClusterUID retrieves the UID of the kube-system namespace using the given Kubernetes clientset. -// This UID can be used as a unique identifier for the Kubernetes cluster. -// The UID is stored in the given context for later retrieval; use ClusterUIDFromContext to get it. -func GetClusterUID(ctx context.Context, clientset kubernetes.Interface) (context.Context, error) { - namespace, err := clientset.CoreV1().Namespaces().Get(ctx, "kube-system", metav1.GetOptions{}) - if err != nil { - return ctx, err - } - - ctx = withClusterUID(ctx, string(namespace.ObjectMeta.UID)) - return ctx, nil -} - -// ClusterUIDFromContext retrieves the cluster UID from the context. -// Panics if the value is not found or if the value is not a string. -func ClusterUIDFromContext(ctx context.Context) string { - value := ctx.Value(clusterUIDKey{}) - if value == nil { - panic("cluster UID not found in context") - } - - uid, ok := value.(string) - if !ok { - panic("cluster UID in context is not a string") - } - - return uid -} - -// withClusterUID adds the given cluster UID to the context -func withClusterUID(ctx context.Context, clusterUID string) context.Context { - return context.WithValue(ctx, clusterUIDKey{}, clusterUID) -} diff --git a/pkg/clusteruid/clusteruid_test.go b/pkg/clusteruid/clusteruid_test.go deleted file mode 100644 index 1d1cacae..00000000 --- a/pkg/clusteruid/clusteruid_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package clusteruid - -import ( - "testing" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/fake" -) - -func TestGetClusterUID(t *testing.T) { - client := fake.NewSimpleClientset() - - mockUID := "12345678-1234-5678-1234-567812345678" - - kubeSystemNS := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-system", - UID: types.UID(mockUID), - }, - } - - _, err := client.CoreV1().Namespaces().Create(t.Context(), kubeSystemNS, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("failed to create kube-system namespace with fake client: %v", err) - } - - ctx, err := GetClusterUID(t.Context(), client) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - uid := ClusterUIDFromContext(ctx) - - if uid != mockUID { - t.Fatalf("expected to get uid=%v, but got uid=%v", mockUID, uid) - } -} diff --git a/pkg/datagatherer/k8s/discovery.go b/pkg/datagatherer/k8s/discovery.go index 586622d6..340bcbda 100644 --- a/pkg/datagatherer/k8s/discovery.go +++ b/pkg/datagatherer/k8s/discovery.go @@ -6,6 +6,7 @@ import ( "k8s.io/client-go/discovery" + "github.com/jetstack/preflight/api" "github.com/jetstack/preflight/pkg/datagatherer" ) @@ -59,15 +60,12 @@ func (g *DataGathererDiscovery) WaitForCacheSync(ctx context.Context) error { // Fetch will fetch discovery data from the apiserver, or return an error func (g *DataGathererDiscovery) Fetch() (interface{}, int, error) { - data, err := g.cl.ServerVersion() + serverVersion, err := g.cl.ServerVersion() if err != nil { return nil, -1, fmt.Errorf("failed to get server version: %v", err) } - response := map[string]interface{}{ - // data has type Info: https://godoc.org/k8s.io/apimachinery/pkg/version#Info - "server_version": data, - } - - return response, len(response), nil + return &api.DiscoveryData{ + ServerVersion: serverVersion, + }, 1, nil } diff --git a/pkg/datagatherer/k8s/dynamic.go b/pkg/datagatherer/k8s/dynamic.go index 50b6c0a4..7cb131b5 100644 --- a/pkg/datagatherer/k8s/dynamic.go +++ b/pkg/datagatherer/k8s/dynamic.go @@ -314,7 +314,6 @@ func (g *DataGathererDynamic) Fetch() (interface{}, int, error) { return nil, -1, fmt.Errorf("resource type must be specified") } - var list = map[string]interface{}{} var items = []*api.GatheredResource{} fetchNamespaces := g.namespaces @@ -344,10 +343,9 @@ func (g *DataGathererDynamic) Fetch() (interface{}, int, error) { return nil, -1, err } - // add gathered resources to items - list["items"] = items - - return list, len(items), nil + return &api.DynamicData{ + Items: items, + }, len(items), nil } // redactList removes sensitive and superfluous data from the supplied resource list. diff --git a/pkg/datagatherer/k8s/dynamic_test.go b/pkg/datagatherer/k8s/dynamic_test.go index 072c4c1c..525c8892 100644 --- a/pkg/datagatherer/k8s/dynamic_test.go +++ b/pkg/datagatherer/k8s/dynamic_test.go @@ -730,15 +730,12 @@ func TestDynamicGatherer_Fetch(t *testing.T) { } if tc.expected != nil { - items, ok := res.(map[string]interface{}) + data, ok := res.(*api.DynamicData) if !ok { - t.Errorf("expected result be an map[string]interface{} but wasn't") + t.Errorf("expected result be *api.DynamicData but wasn't") } - list, ok := items["items"].([]*api.GatheredResource) - if !ok { - t.Errorf("expected result be an []*api.GatheredResource but wasn't") - } + list := data.Items // sorting list of results by name sortGatheredResources(list) // sorting list of expected results by name @@ -1045,10 +1042,9 @@ func TestDynamicGathererNativeResources_Fetch(t *testing.T) { } if tc.expected != nil { - res, ok := rawRes.(map[string]interface{}) - require.Truef(t, ok, "expected result be an map[string]interface{} but wasn't") - actual := res["items"].([]*api.GatheredResource) - require.Truef(t, ok, "expected result be an []*api.GatheredResource but wasn't") + res, ok := rawRes.(*api.DynamicData) + require.Truef(t, ok, "expected result be an *api.DynamicData but wasn't") + actual := res.Items // sorting list of results by name sortGatheredResources(actual) diff --git a/pkg/internal/cyberark/client.go b/pkg/internal/cyberark/client.go new file mode 100644 index 00000000..36d9202b --- /dev/null +++ b/pkg/internal/cyberark/client.go @@ -0,0 +1,72 @@ +package cyberark + +import ( + "context" + "errors" + "net/http" + "os" + + "github.com/jetstack/preflight/pkg/internal/cyberark/dataupload" + "github.com/jetstack/preflight/pkg/internal/cyberark/identity" + "github.com/jetstack/preflight/pkg/internal/cyberark/servicediscovery" +) + +// ClientConfig holds the configuration needed to initialize a CyberArk client. +type ClientConfig struct { + Subdomain string + Username string + Secret string +} + +// ClientConfigLoader is a function type that loads and returns a ClientConfig. +type ClientConfigLoader func() (ClientConfig, error) + +// ErrMissingEnvironmentVariables is returned when required environment variables are not set. +var ErrMissingEnvironmentVariables = errors.New("missing environment variables: ARK_SUBDOMAIN, ARK_USERNAME, ARK_SECRET") + +// LoadClientConfigFromEnvironment loads the CyberArk client configuration from environment variables. +// It expects the following environment variables to be set: +// - ARK_SUBDOMAIN: The CyberArk subdomain to use. +// - ARK_USERNAME: The username for authentication. +// - ARK_SECRET: The secret for authentication. +func LoadClientConfigFromEnvironment() (ClientConfig, error) { + subdomain := os.Getenv("ARK_SUBDOMAIN") + username := os.Getenv("ARK_USERNAME") + secret := os.Getenv("ARK_SECRET") + + if subdomain == "" || username == "" || secret == "" { + return ClientConfig{}, ErrMissingEnvironmentVariables + } + + return ClientConfig{ + Subdomain: subdomain, + Username: username, + Secret: secret, + }, nil + +} + +// NewDatauploadClient initializes and returns a new CyberArk Data Upload client. +// It performs service discovery to find the necessary API endpoints and authenticates +// using the provided client configuration. +func NewDatauploadClient(ctx context.Context, httpClient *http.Client, cfg ClientConfig) (*dataupload.CyberArkClient, error) { + discoveryClient := servicediscovery.New(httpClient) + serviceMap, err := discoveryClient.DiscoverServices(ctx, cfg.Subdomain) + if err != nil { + return nil, err + } + identityAPI := serviceMap.Identity.API + if identityAPI == "" { + return nil, errors.New("service discovery returned an empty identity API") + } + identityClient := identity.New(httpClient, identityAPI, cfg.Subdomain) + err = identityClient.LoginUsernamePassword(ctx, cfg.Username, []byte(cfg.Secret)) + if err != nil { + return nil, err + } + discoveryAPI := serviceMap.DiscoveryContext.API + if discoveryAPI == "" { + return nil, errors.New("service discovery returned an empty discovery API") + } + return dataupload.New(httpClient, discoveryAPI, identityClient.AuthenticateRequest), nil +} diff --git a/pkg/internal/cyberark/client_test.go b/pkg/internal/cyberark/client_test.go new file mode 100644 index 00000000..c8fa98ab --- /dev/null +++ b/pkg/internal/cyberark/client_test.go @@ -0,0 +1,80 @@ +package cyberark_test + +import ( + "crypto/x509" + "errors" + "testing" + + "github.com/jetstack/venafi-connection-lib/http_client" + "github.com/stretchr/testify/require" + "k8s.io/client-go/transport" + "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" + + "github.com/jetstack/preflight/pkg/internal/cyberark" + "github.com/jetstack/preflight/pkg/internal/cyberark/dataupload" + "github.com/jetstack/preflight/pkg/internal/cyberark/servicediscovery" + "github.com/jetstack/preflight/pkg/testutil" + "github.com/jetstack/preflight/pkg/version" + + _ "k8s.io/klog/v2/ktesting/init" +) + +// TestCyberArkClient_PutSnapshot_MockAPI demonstrates that NewDatauploadClient works with the mock API. +func TestCyberArkClient_PutSnapshot_MockAPI(t *testing.T) { + logger := ktesting.NewLogger(t, ktesting.DefaultConfig) + ctx := klog.NewContext(t.Context(), logger) + + httpClient := testutil.FakeCyberArk(t) + + cfg := cyberark.ClientConfig{ + Subdomain: servicediscovery.MockDiscoverySubdomain, + Username: "test@example.com", + Secret: "somepassword", + } + + cl, err := cyberark.NewDatauploadClient(ctx, httpClient, cfg) + require.NoError(t, err) + + err = cl.PutSnapshot(ctx, dataupload.Snapshot{ + ClusterID: "success-cluster-id", + }) + require.NoError(t, err) +} + +// TestCyberArkClient_PutSnapshot_RealAPI demonstrates that NewDatauploadClient works with the real inventory API. +// +// An API token is obtained by authenticating with the ARK_USERNAME and ARK_SECRET from the environment. +// ARK_SUBDOMAIN should be your tenant subdomain. +// +// To test against a tenant on the integration platform, also set: +// ARK_DISCOVERY_API=https://platform-discovery.integration-cyberark.cloud/api/v2 +// +// To enable verbose request logging: +// +// go test ./pkg/internal/cyberark \ +// -v -count 1 -run TestCyberArkClient_PutSnapshot_RealAPI -args -testing.v 6 +func TestCyberArkClient_PutSnapshot_RealAPI(t *testing.T) { + logger := ktesting.NewLogger(t, ktesting.DefaultConfig) + ctx := klog.NewContext(t.Context(), logger) + + var rootCAs *x509.CertPool + httpClient := http_client.NewDefaultClient(version.UserAgent(), rootCAs) + httpClient.Transport = transport.NewDebuggingRoundTripper(httpClient.Transport, transport.DebugByContext) + + cfg, err := cyberark.LoadClientConfigFromEnvironment() + if err != nil { + if errors.Is(err, cyberark.ErrMissingEnvironmentVariables) { + t.Skipf("Skipping: %s", err) + } + require.NoError(t, err) + } + + cl, err := cyberark.NewDatauploadClient(ctx, httpClient, cfg) + require.NoError(t, err) + + err = cl.PutSnapshot(ctx, dataupload.Snapshot{ + ClusterID: "bb068932-c80d-460d-88df-34bc7f3f3297", + }) + require.NoError(t, err) +} diff --git a/pkg/internal/cyberark/dataupload/dataupload.go b/pkg/internal/cyberark/dataupload/dataupload.go index 82fdd4b1..9d9444f8 100644 --- a/pkg/internal/cyberark/dataupload/dataupload.go +++ b/pkg/internal/cyberark/dataupload/dataupload.go @@ -35,10 +35,6 @@ type CyberArkClient struct { authenticateRequest func(req *http.Request) error } -type Options struct { - ClusterName string -} - func New(httpClient *http.Client, baseURL string, authenticateRequest func(req *http.Request) error) *CyberArkClient { return &CyberArkClient{ baseURL: baseURL, diff --git a/pkg/internal/cyberark/dataupload/dataupload_test.go b/pkg/internal/cyberark/dataupload/dataupload_test.go index 9df78244..4c278e20 100644 --- a/pkg/internal/cyberark/dataupload/dataupload_test.go +++ b/pkg/internal/cyberark/dataupload/dataupload_test.go @@ -1,22 +1,15 @@ package dataupload_test import ( - "crypto/x509" "fmt" "net/http" - "os" "testing" - "github.com/jetstack/venafi-connection-lib/http_client" "github.com/stretchr/testify/require" - "k8s.io/client-go/transport" "k8s.io/klog/v2" "k8s.io/klog/v2/ktesting" "github.com/jetstack/preflight/pkg/internal/cyberark/dataupload" - "github.com/jetstack/preflight/pkg/internal/cyberark/identity" - "github.com/jetstack/preflight/pkg/internal/cyberark/servicediscovery" - "github.com/jetstack/preflight/pkg/version" _ "k8s.io/klog/v2/ktesting/init" ) @@ -109,47 +102,3 @@ func TestCyberArkClient_PutSnapshot_MockAPI(t *testing.T) { }) } } - -// TestCyberArkClient_PutSnapshot_RealAPI demonstrates that the dataupload code works with the real inventory API. -// An API token is obtained by authenticating with the ARK_USERNAME and ARK_SECRET from the environment. -// ARK_SUBDOMAIN should be your tenant subdomain. -// -// To test against a tenant on the integration platform, also set: -// ARK_DISCOVERY_API=https://platform-discovery.integration-cyberark.cloud/api/v2 -// -// To enable verbose request logging: -// -// go test ./pkg/internal/cyberark/dataupload/... \ -// -v -count 1 -run TestCyberArkClient_PutSnapshot_RealAPI -args -testing.v 6 -func TestCyberArkClient_PutSnapshot_RealAPI(t *testing.T) { - subdomain := os.Getenv("ARK_SUBDOMAIN") - username := os.Getenv("ARK_USERNAME") - secret := os.Getenv("ARK_SECRET") - - if subdomain == "" || username == "" || secret == "" { - t.Skip("Skipping because one of the following environment variables is unset or empty: ARK_SUBDOMAIN, ARK_USERNAME, ARK_SECRET") - return - } - - logger := ktesting.NewLogger(t, ktesting.DefaultConfig) - ctx := klog.NewContext(t.Context(), logger) - - var rootCAs *x509.CertPool - httpClient := http_client.NewDefaultClient(version.UserAgent(), rootCAs) - httpClient.Transport = transport.NewDebuggingRoundTripper(httpClient.Transport, transport.DebugByContext) - - discoveryClient := servicediscovery.New(httpClient) - - services, err := discoveryClient.DiscoverServices(ctx, subdomain) - require.NoError(t, err) - - identityClient := identity.New(httpClient, services.Identity.API, subdomain) - err = identityClient.LoginUsernamePassword(ctx, username, []byte(secret)) - require.NoError(t, err) - - cyberArkClient := dataupload.New(httpClient, services.DiscoveryContext.API, identityClient.AuthenticateRequest) - err = cyberArkClient.PutSnapshot(ctx, dataupload.Snapshot{ - ClusterID: "bb068932-c80d-460d-88df-34bc7f3f3297", - }) - require.NoError(t, err) -} diff --git a/pkg/internal/cyberark/dataupload/mock.go b/pkg/internal/cyberark/dataupload/mock.go index 55054388..6fc74598 100644 --- a/pkg/internal/cyberark/dataupload/mock.go +++ b/pkg/internal/cyberark/dataupload/mock.go @@ -123,7 +123,7 @@ func (mds *mockDataUploadServer) handleSnapshotLinks(w http.ResponseWriter, r *h } if req.ClusterID != successClusterID { - http.Error(w, "post body contains cluster ID", http.StatusInternalServerError) + http.Error(w, "post body does not contain cluster ID", http.StatusInternalServerError) return } diff --git a/pkg/internal/cyberark/identity/mock.go b/pkg/internal/cyberark/identity/mock.go index 81854f44..d904a695 100644 --- a/pkg/internal/cyberark/identity/mock.go +++ b/pkg/internal/cyberark/identity/mock.go @@ -28,7 +28,7 @@ const ( // mockSuccessfulStartAuthenticationToken is the token returned by the // mock server in response to a successful AdvanceAuthentication request // Must match what's in testdata/advance_authentication_success.json - mockSuccessfulStartAuthenticationToken = "long-token" + mockSuccessfulStartAuthenticationToken = "success-token" ) var ( diff --git a/pkg/internal/cyberark/identity/testdata/advance_authentication_success.json b/pkg/internal/cyberark/identity/testdata/advance_authentication_success.json index c35ed116..62495358 100644 --- a/pkg/internal/cyberark/identity/testdata/advance_authentication_success.json +++ b/pkg/internal/cyberark/identity/testdata/advance_authentication_success.json @@ -3,7 +3,7 @@ "Result": { "AuthLevel": "Normal", "DisplayName": "Namey McNamerson", - "Token": "long-token", + "Token": "success-token", "Auth": "auth-auth", "UserId": "11111111-2222-3333-4444-555555555555", "EmailAddress": "name@example.com", diff --git a/pkg/internal/cyberark/servicediscovery/discovery.go b/pkg/internal/cyberark/servicediscovery/discovery.go index c2cad928..9aeab674 100644 --- a/pkg/internal/cyberark/servicediscovery/discovery.go +++ b/pkg/internal/cyberark/servicediscovery/discovery.go @@ -16,9 +16,13 @@ const ( // ProdDiscoveryAPIBaseURL is the base URL for the production CyberArk Service Discovery API ProdDiscoveryAPIBaseURL = "https://platform-discovery.cyberark.cloud/api/v2/" - // identityServiceName is the name of the identity service we're looking for in responses from the Service Discovery API + // IdentityServiceName is the name of the identity service we're looking for in responses from the Service Discovery API // We were told to use the identity_administration field, not the identity_user_portal field. - identityServiceName = "identity_administration" + IdentityServiceName = "identity_administration" + + // DiscoveryContextServiceName is the name of the discovery and context API + // in responses from the Service Discovery API. + DiscoveryContextServiceName = "discoverycontext" // maxDiscoverBodySize is the maximum allowed size for a response body from the CyberArk Service Discovery subdomain endpoint // As of 2025-04-16, a response from the integration environment is ~4kB @@ -101,7 +105,6 @@ func (c *Client) DiscoverServices(ctx context.Context, subdomain string) (*Servi } var services Services - err = json.NewDecoder(io.LimitReader(resp.Body, maxDiscoverBodySize)).Decode(&services) if err != nil { if err == io.ErrUnexpectedEOF { @@ -112,7 +115,7 @@ func (c *Client) DiscoverServices(ctx context.Context, subdomain string) (*Servi } if services.Identity.API == "" { - return nil, fmt.Errorf("didn't find %s in service discovery response, which may indicate a suspended tenant; unable to detect CyberArk Identity API URL", identityServiceName) + return nil, fmt.Errorf("didn't find %s in service discovery response, which may indicate a suspended tenant; unable to detect CyberArk Identity API URL", IdentityServiceName) } return &services, nil diff --git a/pkg/internal/cyberark/servicediscovery/discovery_test.go b/pkg/internal/cyberark/servicediscovery/discovery_test.go index cc1900f2..d1091307 100644 --- a/pkg/internal/cyberark/servicediscovery/discovery_test.go +++ b/pkg/internal/cyberark/servicediscovery/discovery_test.go @@ -31,7 +31,7 @@ func Test_DiscoverIdentityAPIURL(t *testing.T) { "no identity service in response": { subdomain: "no-identity", expectedURL: "", - expectedError: fmt.Errorf("didn't find %s in service discovery response, which may indicate a suspended tenant; unable to detect CyberArk Identity API URL", identityServiceName), + expectedError: fmt.Errorf("didn't find %s in service discovery response, which may indicate a suspended tenant; unable to detect CyberArk Identity API URL", IdentityServiceName), }, "unexpected HTTP response": { subdomain: "bad-request", diff --git a/pkg/internal/cyberark/servicediscovery/mock.go b/pkg/internal/cyberark/servicediscovery/mock.go index 87d8ef36..12607903 100644 --- a/pkg/internal/cyberark/servicediscovery/mock.go +++ b/pkg/internal/cyberark/servicediscovery/mock.go @@ -48,7 +48,7 @@ type mockDiscoveryServer struct { // // The returned HTTP client has a transport which logs requests and responses // depending on log level of the logger supplied in the context. -func MockDiscoveryServer(t *testing.T, services Services) *http.Client { +func MockDiscoveryServer(t testing.TB, services Services) *http.Client { tmpl := template.Must(template.New("mockDiscoverySuccess").Parse(discoverySuccessTemplate)) buf := &bytes.Buffer{} err := tmpl.Execute(buf, services) diff --git a/pkg/testutil/datareadings.go b/pkg/testutil/datareadings.go new file mode 100644 index 00000000..9b281325 --- /dev/null +++ b/pkg/testutil/datareadings.go @@ -0,0 +1,90 @@ +package testutil + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/jetstack/preflight/api" +) + +// ParseDataReadings decodes JSON encoded datareadings. +// It attempts to decode the data of each reading into a concrete type. +// It tries to decode the data as DynamicData and DiscoveryData and then gives +// up with a test failure. +// This function is useful for reading sample datareadings from disk for use in +// CyberArk dataupload client tests, which require the datareadings data to have +// rich types +// TODO(wallrj): Refactor this so that it can be used with the `agent +// --input-path` feature, to enable datareadings to be read from disk and pushed +// to CyberArk. +func ParseDataReadings(t *testing.T, data []byte) []*api.DataReading { + var dataReadings []*api.DataReading + + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + err := decoder.Decode(&dataReadings) + require.NoError(t, err) + + for _, reading := range dataReadings { + dataBytes, err := json.Marshal(reading.Data) + require.NoError(t, err) + in := bytes.NewReader(dataBytes) + d := json.NewDecoder(in) + d.DisallowUnknownFields() + + var dynamicGatherData api.DynamicData + if err := d.Decode(&dynamicGatherData); err == nil { + reading.Data = &dynamicGatherData + continue + } + + _, err = in.Seek(0, 0) + require.NoError(t, err) + + var discoveryData api.DiscoveryData + if err = d.Decode(&discoveryData); err == nil { + reading.Data = &discoveryData + continue + } + + require.Failf(t, "failed to parse reading", "reading: %#v", reading) + } + return dataReadings +} + +// ReadGZIP Reads the gzip file at path, and returns the decompressed bytes +func ReadGZIP(t *testing.T, path string) []byte { + f, err := os.Open(path) + require.NoError(t, err) + defer func() { require.NoError(t, f.Close()) }() + gzr, err := gzip.NewReader(f) + require.NoError(t, err) + defer func() { require.NoError(t, gzr.Close()) }() + bytes, err := io.ReadAll(gzr) + require.NoError(t, err) + return bytes +} + +// WriteGZIP writes gzips the data and writes it to path. +func WriteGZIP(t *testing.T, path string, data []byte) { + tmp, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".*") + require.NoError(t, err) + gzw := gzip.NewWriter(tmp) + _, err = gzw.Write(data) + require.NoError(t, errors.Join( + err, + gzw.Flush(), + gzw.Close(), + tmp.Close(), + )) + err = os.Rename(tmp.Name(), path) + require.NoError(t, err) +} diff --git a/pkg/testutil/envtest.go b/pkg/testutil/envtest.go index 149368f5..73251ec6 100644 --- a/pkg/testutil/envtest.go +++ b/pkg/testutil/envtest.go @@ -26,6 +26,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" "github.com/jetstack/preflight/pkg/client" + "github.com/jetstack/preflight/pkg/internal/cyberark/dataupload" + "github.com/jetstack/preflight/pkg/internal/cyberark/identity" + "github.com/jetstack/preflight/pkg/internal/cyberark/servicediscovery" ) // To see the API server logs, set: @@ -259,6 +262,33 @@ func FakeTPP(t testing.TB) (*httptest.Server, *x509.Certificate) { return server, cert } +// FakeCyberArk returns an HTTP client that will route requests to mock CyberArk +// Service Discovery, Identity and Discovery and Context APIs. This is useful +// for testing code that uses all those APIs, such as +// `cyberark.NewDatauploadClient`. +// +// The environment variable `ARK_DISCOVERY_API` is set to the URL of the mock +// Service Discovery API, for the supplied `testing.TB` so that the client under +// test will use the mock Service Discovery API. +// +// The returned HTTP client has a transport which logs requests and responses +// depending on log level of the logger supplied in the context. +func FakeCyberArk(t testing.TB) *http.Client { + t.Helper() + + identityAPI, _ := identity.MockIdentityServer(t) + discoveryContextAPI, _ := dataupload.MockDataUploadServer(t) + httpClient := servicediscovery.MockDiscoveryServer(t, servicediscovery.Services{ + Identity: servicediscovery.ServiceEndpoint{ + API: identityAPI, + }, + DiscoveryContext: servicediscovery.ServiceEndpoint{ + API: discoveryContextAPI, + }, + }) + return httpClient +} + // Generated using: // // helm template ./deploy/charts/venafi-kubernetes-agent -n venafi --set crds.venafiConnection.include=true --show-only templates/venafi-connection-rbac.yaml | grep -ivE '(helm|\/version)'