Skip to content

Use upstream install.Deployment, use secretName from BSL spec for volumeMount/Volume names #234

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dymurray merged 6 commits into from
Sep 1, 2021
Merged

Use upstream install.Deployment, use secretName from BSL spec for volumeMount/Volume names #234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c24045a
use upstream install.Deployment
kaovilai Aug 25, 2021
a8b9bb2
nil check for bsl.Credential.Name
kaovilai Aug 26, 2021
a1e3c4f
continue ignore nil check
kaovilai Aug 26, 2021
c291880
resolve discussions
kaovilai Aug 31, 2021
bfebabb
When you do this please also update the test
kaovilai Aug 31, 2021
a9e7977
PullAlways velero iamge
kaovilai Aug 31, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion api/v1alpha1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

226 changes: 88 additions & 138 deletions controllers/velero.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"fmt"
"os"
"reflect"
"strings"

"github.com/openshift/oadp-operator/pkg/credentials"
v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
Expand Down Expand Up @@ -52,6 +51,15 @@ const (
RegistryImage = "quay.io/konveyor/registry:latest"
)

var (
veleroLabelSelector = &metav1.LabelSelector{
MatchLabels: map[string]string{
"component": common.Velero,
"deploy": common.Velero,
},
}
)

func (r *VeleroReconciler) ReconcileVeleroServiceAccount(log logr.Logger) (bool, error) {
velero := oadpv1alpha1.Velero{}
if err := r.Get(r.Context, r.NamespacedName, &velero); err != nil {
Expand Down Expand Up @@ -256,11 +264,7 @@ func (r *VeleroReconciler) ReconcileVeleroDeployment(log logr.Logger) (bool, err

// Setting Deployment selector if a new object is created as it is immutable
if veleroDeployment.ObjectMeta.CreationTimestamp.IsZero() {
veleroDeployment.Spec.Selector = &metav1.LabelSelector{
MatchLabels: map[string]string{
"component": common.Velero,
},
}
veleroDeployment.Spec.Selector = veleroLabelSelector
}

// Setting controller owner reference on the velero deployment
Expand Down Expand Up @@ -359,133 +363,6 @@ func (r *VeleroReconciler) buildVeleroDeployment(veleroDeployment *appsv1.Deploy
return fmt.Errorf("velero deployment cannot be nil")
}

// get default values
volumeMounts := []corev1.VolumeMount{
{
Name: "plugins",
MountPath: "/plugins",
},
{
Name: "scratch",
MountPath: "/scratch",
},
{
Name: "certs",
MountPath: "/etc/ssl/certs",
},
}

envVars := []corev1.EnvVar{
{
Name: common.LDLibraryPathEnvKey,
Value: "/plugins",
},
{
Name: common.VeleroNamespaceEnvKey,
Value: velero.Namespace,
},
{
Name: common.VeleroScratchDirEnvKey,
Value: "/scratch",
},
{
Name: common.HTTPProxyEnvVar,
Value: os.Getenv("HTTP_PROXY"),
},
{
Name: common.HTTPSProxyEnvVar,
Value: os.Getenv("HTTPS_PROXY"),
},
{
Name: common.NoProxyEnvVar,
Value: os.Getenv("NO_PROXY"),
},
}

volumes := []corev1.Volume{
{
Name: "plugins",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
{
Name: "scratch",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
{
Name: "certs",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
}

//add any default init containers here if needed eg: setup-certificate-secret
initContainers := []corev1.Container{}

// assign spec values
veleroDeployment.Labels = r.getAppLabels(velero)

veleroDeployment.Spec = appsv1.DeploymentSpec{
//TODO: add velero nodeselector, needs to be added to the VELERO CR first
Selector: veleroDeployment.Spec.Selector,
Replicas: pointer.Int32(1),
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"component": common.Velero,
},
Annotations: map[string]string{
"prometheus.io/scrape": "true",
"prometheus.io/port": "8085",
"prometheus.io/path": "/metrics",
},
},
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyAlways,
ServiceAccountName: common.Velero,
Tolerations: velero.Spec.VeleroTolerations,
Containers: []corev1.Container{
r.buildVeleroContainer(velero, volumeMounts, envVars),
},
Volumes: volumes,
InitContainers: initContainers,
},
},
}

err := credentials.AppendPluginSpecificSpecs(velero, veleroDeployment)

if err != nil {
return err
}

return nil
}

func (r *VeleroReconciler) buildVeleroContainer(velero *oadpv1alpha1.Velero, volumeMounts []corev1.VolumeMount, envVars []corev1.EnvVar) corev1.Container {
container := corev1.Container{
Name: common.Velero,
Image: getVeleroImage(),
ImagePullPolicy: corev1.PullAlways,
Ports: []corev1.ContainerPort{
{
Name: "metrics",
ContainerPort: 8085,
},
},
Resources: r.getVeleroResourceReqs(velero),
Command: []string{"/velero"},
//TODO: Parametrize VELERO debug flag
Args: []string{
"server",
},
VolumeMounts: volumeMounts,
Env: envVars,
}
//check if CSI plugin is added in spec
for _, plugin := range velero.Spec.DefaultVeleroPlugins {
if plugin == oadpv1alpha1.DefaultPluginCSI {
Expand All @@ -503,19 +380,92 @@ func (r *VeleroReconciler) buildVeleroContainer(velero *oadpv1alpha1.Velero, vol
break
}
}
deploymentName := veleroDeployment.Name //saves desired deployment name before install.Deployment overwrites them.
Comment thread
kaovilai marked this conversation as resolved.
*veleroDeployment = *install.Deployment(veleroDeployment.Namespace,
Comment thread
dymurray marked this conversation as resolved.
install.WithResources(r.getVeleroResourceReqs(velero)),
install.WithImage(getVeleroImage()),
install.WithFeatures(velero.Spec.VeleroFeatureFlags),
// use WithSecret false even if we have secret because we use a different VolumeMounts and EnvVars
Comment thread
kaovilai marked this conversation as resolved.
// see: https://github.com/vmware-tanzu/velero/blob/ed5809b7fc22f3661eeef10bdcb63f0d74472b76/pkg/install/deployment.go#L223-L261
// our secrets are appended to containers/volumeMounts in credentials.AppendPluginSpecificSpecs function
install.WithSecret(false),
)
// adjust veleroDeployment from install
veleroDeployment.Name = deploymentName //reapply saved deploymentName
return r.customizeVeleroDeployment(velero, veleroDeployment)
}

func (r *VeleroReconciler) customizeVeleroDeployment(velero *oadpv1alpha1.Velero, veleroDeployment *appsv1.Deployment) error {
veleroDeployment.Labels = r.getAppLabels(velero)
veleroDeployment.Spec.Selector = veleroLabelSelector

//TODO: add velero nodeselector, needs to be added to the VELERO CR first
Comment thread
kaovilai marked this conversation as resolved.
// Selector: veleroDeployment.Spec.Selector,
veleroDeployment.Spec.Replicas = pointer.Int32(1)
veleroDeployment.Spec.Template.Spec.Tolerations = velero.Spec.VeleroTolerations
veleroDeployment.Spec.Template.Spec.Volumes = append(veleroDeployment.Spec.Template.Spec.Volumes,
corev1.Volume{
Copy link
Copy Markdown
Member

@dymurray dymurray Aug 31, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@shubham-pampattiwar can you remind me, do we need this? I thought maybe we determined this came from an older instance of our installation we no longer needed. Setting this to an emptydir volume doesn't seem to be doing anything... but I can't remember what we said about it.

Copy link
Copy Markdown
Member

@shubham-pampattiwar shubham-pampattiwar Aug 31, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dymurray I am not sure if this is needed but would like to keep it for now 😅 , IIRC we removed the certs-* initContainer.

Name: "certs",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
})
//add any default init containers here if needed eg: setup-certificate-secret
Comment thread
kaovilai marked this conversation as resolved.
// When you do this
// - please set the ImagePullPolicy to Always, and
// - please also update the test
if veleroDeployment.Spec.Template.Spec.InitContainers == nil {
veleroDeployment.Spec.Template.Spec.InitContainers = []corev1.Container{}
}

var veleroContainer *corev1.Container
for i, container := range veleroDeployment.Spec.Template.Spec.Containers {
if container.Name == common.Velero {
veleroContainer = &veleroDeployment.Spec.Template.Spec.Containers[i]
break
}
}
if err := r.customizeVeleroContainer(velero, veleroDeployment, veleroContainer); err != nil {
return err
}
return credentials.AppendPluginSpecificSpecs(velero, veleroDeployment, veleroContainer)
Comment thread
kaovilai marked this conversation as resolved.
}

//Append EnabledFeaturesFlags to velero container
if len(velero.Spec.VeleroFeatureFlags) > 0 {
container.Args = append(container.Args, fmt.Sprintf("--features=%s", strings.Join(velero.Spec.VeleroFeatureFlags, ",")))
func (r *VeleroReconciler) customizeVeleroContainer(velero *oadpv1alpha1.Velero, veleroDeployment *appsv1.Deployment, veleroContainer *corev1.Container) error {
if veleroContainer == nil {
return fmt.Errorf("could not find velero container in Deployment")
}
veleroContainer.ImagePullPolicy = corev1.PullAlways
veleroContainer.VolumeMounts = append(veleroContainer.VolumeMounts,
corev1.VolumeMount{
Name: "certs",
MountPath: "/etc/ssl/certs",
},
)

veleroContainer.Env = append(veleroContainer.Env,
corev1.EnvVar{
Name: common.HTTPProxyEnvVar,
Value: os.Getenv("HTTP_PROXY"),
},
corev1.EnvVar{
Name: common.HTTPSProxyEnvVar,
Value: os.Getenv("HTTPS_PROXY"),
},
corev1.EnvVar{
Name: common.NoProxyEnvVar,
Value: os.Getenv("NO_PROXY"),
},
)
// Enable user to specify --restic-timeout (defaults to 1h)
resticTimeout := "1h"
if len(velero.Spec.ResticTimeout) > 0 {
resticTimeout = velero.Spec.ResticTimeout
}
container.Args = append(container.Args, fmt.Sprintf("--restic-timeout=%s", resticTimeout))
return container
// Append restic timeout option manually. Not configurable via install package, missing from podTemplateConfig struct. See: https://github.com/vmware-tanzu/velero/blob/8d57215ded1aa91cdea2cf091d60e072ce3f340f/pkg/install/deployment.go#L34-L45
veleroContainer.Args = append(veleroContainer.Args, fmt.Sprintf("--restic-timeout=%s", resticTimeout))
Comment thread
kaovilai marked this conversation as resolved.

return nil
}

func getVeleroImage() string {
Expand Down
Loading