From 58076e7c4d417be40df24a290eef6a08843217d1 Mon Sep 17 00:00:00 2001 From: Rahul Gidwani Date: Mon, 6 Feb 2023 10:10:50 -0800 Subject: [PATCH 1/2] Upgrading fabric k8s client for k8s overlord extensions --- .../kubernetes-overlord-extensions/pom.xml | 14 ++++++++++---- .../overlord/common/DruidKubernetesPeonClient.java | 13 +++++++++---- .../druid/k8s/overlord/common/K8sTestUtils.java | 8 ++++++++ .../common/SingleContainerTaskAdapterTest.java | 12 ++---------- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/extensions-contrib/kubernetes-overlord-extensions/pom.xml b/extensions-contrib/kubernetes-overlord-extensions/pom.xml index 6083a0d53fc8..fcabd56c2a1a 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/pom.xml +++ b/extensions-contrib/kubernetes-overlord-extensions/pom.xml @@ -104,7 +104,7 @@ io.fabric8 kubernetes-model-core - 5.12.2 + 6.4.1 javax.validation @@ -114,12 +114,12 @@ io.fabric8 kubernetes-model-batch - 5.12.2 + 6.4.1 io.fabric8 kubernetes-client - 5.12.2 + 6.4.1 @@ -136,7 +136,7 @@ io.fabric8 kubernetes-server-mock - 5.12.2 + 6.4.1 test @@ -226,6 +226,12 @@ equalsverifier test + + io.kubernetes + client-java-proto + 11.0.1 + compile + diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/DruidKubernetesPeonClient.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/DruidKubernetesPeonClient.java index e6fe2eaa84d1..af9a58694ac7 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/DruidKubernetesPeonClient.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/DruidKubernetesPeonClient.java @@ -78,7 +78,7 @@ public Pod launchJobAndWaitForStart(Job job, long howLong, TimeUnit timeUnit) long start = System.currentTimeMillis(); // launch job return clientApi.executeRequest(client -> { - client.batch().v1().jobs().inNamespace(namespace).create(job); + client.batch().v1().jobs().inNamespace(namespace).withName(job.getMetadata().getName()).create(); K8sTaskId taskId = new K8sTaskId(job.getMetadata().getName()); log.info("Successfully submitted job: %s ... waiting for job to launch", taskId); // wait until the pod is running or complete or failed, any of those is fine @@ -121,12 +121,12 @@ public JobResponse waitForJobCompletion(K8sTaskId taskId, long howLong, TimeUnit public boolean cleanUpJob(K8sTaskId taskId) { if (!debugJobs) { - Boolean result = clientApi.executeRequest(client -> client.batch() + Boolean result = clientApi.executeRequest(client -> !client.batch() .v1() .jobs() .inNamespace(namespace) .withName(taskId.getK8sTaskId()) - .delete()); + .delete().isEmpty()); if (result) { log.info("Cleaned up k8s task: %s", taskId); } else { @@ -216,7 +216,12 @@ public int cleanCompletedJobsOlderThan(long howFarBack, TimeUnit timeUnit) return clientApi.executeRequest(client -> { List jobs = getJobsToCleanup(listAllPeonJobs(), howFarBack, timeUnit); jobs.forEach(x -> { - if (client.batch().v1().jobs().inNamespace(namespace).withName(x.getMetadata().getName()).delete()) { + if (!client.batch() + .v1() + .jobs() + .inNamespace(namespace) + .withName(x.getMetadata().getName()) + .delete().isEmpty()) { numDeleted.incrementAndGet(); } }); diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java index 0f454102986d..503708f77430 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java @@ -27,6 +27,7 @@ import io.fabric8.kubernetes.api.model.PodSpecBuilder; import io.fabric8.kubernetes.api.model.PodTemplateSpec; import io.fabric8.kubernetes.api.model.batch.v1.Job; +import io.fabric8.kubernetes.client.utils.Serialization; import org.apache.commons.text.CharacterPredicates; import org.apache.commons.text.RandomStringGenerator; import org.apache.druid.data.input.impl.DimensionsSpec; @@ -142,4 +143,11 @@ public static Task getTask() ); } + public static T fileToResource(String contents, Class type) + { + return Serialization.unmarshal( + MultiContainerTaskAdapter.class.getClassLoader().getResourceAsStream(contents), + type + ); + } } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/SingleContainerTaskAdapterTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/SingleContainerTaskAdapterTest.java index d19ac42f5e92..3ae20f35cb69 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/SingleContainerTaskAdapterTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/SingleContainerTaskAdapterTest.java @@ -65,9 +65,7 @@ public SingleContainerTaskAdapterTest() public void testSingleContainerSupport() throws IOException { TestKubernetesClient testClient = new TestKubernetesClient(client); - Pod pod = client.pods() - .load(this.getClass().getClassLoader().getResourceAsStream("multiContainerPodSpec.yaml")) - .get(); + Pod pod = K8sTestUtils.fileToResource("multiContainerPodSpec.yaml", Pod.class); KubernetesTaskRunnerConfig config = new KubernetesTaskRunnerConfig(); config.namespace = "test"; SingleContainerTaskAdapter adapter = new SingleContainerTaskAdapter(testClient, config, jsonMapper); @@ -81,13 +79,7 @@ public void testSingleContainerSupport() throws IOException new File("/tmp") ) ); - Job expected = client.batch() - .v1() - .jobs() - .load(this.getClass() - .getClassLoader() - .getResourceAsStream("expectedSingleiContainerOutput.yaml")) - .get(); + Job expected = K8sTestUtils.fileToResource("expectedSingleiContainerOutput.yaml", Job.class); // something is up with jdk 17, where if you compress with jdk < 17 and try and decompress you get different results, // this would never happen in real life, but for the jdk 17 tests this is a problem // could be related to: https://bugs.openjdk.org/browse/JDK-8081450 From a785fa905b3530cc5234ca677d87eed84c0036e0 Mon Sep 17 00:00:00 2001 From: Rahul Gidwani Date: Mon, 6 Feb 2023 11:03:28 -0800 Subject: [PATCH 2/2] Forgot to bump the license file --- licenses.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/licenses.yaml b/licenses.yaml index ba21f0b0456d..bf3beff5370f 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -901,7 +901,7 @@ name: kubernetes fabric java client license_category: binary module: extensions-contrib/kubernetes-overlord-extensions license_name: Apache License version 2.0 -version: 5.12.2 +version: 6.4.1 libraries: - io.fabric8: kubernetes-client