From f919092147618069e4ee1aa10efac41c15566c1a Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 13 May 2025 09:26:00 +0200 Subject: [PATCH 1/5] HDDS-13024. HTTP connect to 0.0.0.0 failed --- .../org/apache/hadoop/hdds/server/http/BaseHttpServer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java index 26457d9d0c68..74e415de037e 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java @@ -354,14 +354,14 @@ public void updateConnectorAddress() { int connIdx = 0; if (policy.isHttpEnabled()) { httpAddress = httpServer.getConnectorAddress(connIdx++); - String realAddress = NetUtils.getHostPortString(httpAddress); + String realAddress = NetUtils.getHostPortString(NetUtils.getConnectAddress(httpAddress)); conf.set(getHttpAddressKey(), realAddress); LOG.info("HTTP server of {} listening at http://{}", name, realAddress); } if (policy.isHttpsEnabled()) { httpsAddress = httpServer.getConnectorAddress(connIdx); - String realAddress = NetUtils.getHostPortString(httpsAddress); + String realAddress = NetUtils.getHostPortString(NetUtils.getConnectAddress(httpsAddress)); conf.set(getHttpsAddressKey(), realAddress); LOG.info("HTTPS server of {} listening at https://{}", name, realAddress); } From b7ae0b5b434c00de547a68fa56f66f64754da869 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 13 May 2025 10:07:04 +0200 Subject: [PATCH 2/5] add unit test --- .../hdds/server/http/TestBaseHttpServer.java | 130 +++++++++++++++++- .../test/resources/webapps/testing/.gitkeep | 0 2 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 hadoop-hdds/framework/src/test/resources/webapps/testing/.gitkeep diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestBaseHttpServer.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestBaseHttpServer.java index 54b93f7464d5..f96c9a521632 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestBaseHttpServer.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestBaseHttpServer.java @@ -19,13 +19,46 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +import java.io.IOException; +import java.net.InetAddress; +import java.nio.file.Path; +import org.apache.commons.lang3.NotImplementedException; +import org.apache.hadoop.hdds.conf.MutableConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.ozone.test.GenericTestUtils.PortAllocator; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; /** * Test Common ozone/hdds web methods. */ -public class TestBaseHttpServer { +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestBaseHttpServer { + + private static final String ADDRESS_HTTP_KEY = "address.http"; + private static final String ADDRESS_HTTPS_KEY = "address.https"; + private static final String BIND_HOST_HTTP_KEY = "bind-host.http"; + private static final String BIND_HOST_HTTPS_KEY = "bind-host.https"; + private static final String BIND_HOST_DEFAULT = "0.0.0.0"; + private static final int BIND_PORT_HTTP_DEFAULT = PortAllocator.getFreePort(); + private static final int BIND_PORT_HTTPS_DEFAULT = PortAllocator.getFreePort(); + private static final String ENABLED_KEY = "enabled"; + + private String hostname; + + @TempDir + private Path tempDir; + + @BeforeAll + void setup() throws Exception { + hostname = InetAddress.getLocalHost().getHostName(); + } + @Test public void getBindAddress() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); @@ -106,4 +139,99 @@ protected String getHttpAuthConfigPrefix() { "default", 65).toString()); } + @ParameterizedTest + @EnumSource + void updatesAddressInConfig(HttpConfig.Policy policy) throws Exception { + MutableConfigurationSource conf = newConfig(policy); + + BaseHttpServer subject = new TestingHttpServer(conf); + + try { + subject.start(); + + if (policy.isHttpEnabled()) { + assertEquals(hostname + ":" + subject.getHttpAddress().getPort(), conf.get(ADDRESS_HTTP_KEY)); + } + if (policy.isHttpsEnabled()) { + assertEquals(hostname + ":" + subject.getHttpsAddress().getPort(), conf.get(ADDRESS_HTTPS_KEY)); + } + } finally { + subject.stop(); + } + } + + private MutableConfigurationSource newConfig(HttpConfig.Policy policy) { + MutableConfigurationSource conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_HTTP_BASEDIR, tempDir.toString()); + conf.setEnum(OzoneConfigKeys.OZONE_HTTP_POLICY_KEY, policy); + return conf; + } + + private static class TestingHttpServer extends BaseHttpServer { + + TestingHttpServer(MutableConfigurationSource conf) throws IOException { + super(conf, "testing"); + } + + @Override + protected String getHttpAddressKey() { + return ADDRESS_HTTP_KEY; + } + + @Override + protected String getHttpsAddressKey() { + return ADDRESS_HTTPS_KEY; + } + + @Override + protected String getHttpBindHostKey() { + return BIND_HOST_HTTP_KEY; + } + + @Override + protected String getHttpsBindHostKey() { + return BIND_HOST_HTTPS_KEY; + } + + @Override + protected String getBindHostDefault() { + return BIND_HOST_DEFAULT; + } + + @Override + protected int getHttpBindPortDefault() { + return BIND_PORT_HTTP_DEFAULT; + } + + @Override + protected int getHttpsBindPortDefault() { + return BIND_PORT_HTTPS_DEFAULT; + } + + @Override + protected String getKeytabFile() { + throw new NotImplementedException(); + } + + @Override + protected String getSpnegoPrincipal() { + throw new NotImplementedException(); + } + + @Override + protected String getEnabledKey() { + return ENABLED_KEY; + } + + @Override + protected String getHttpAuthType() { + throw new NotImplementedException(); + } + + @Override + protected String getHttpAuthConfigPrefix() { + throw new NotImplementedException(); + } + } + } diff --git a/hadoop-hdds/framework/src/test/resources/webapps/testing/.gitkeep b/hadoop-hdds/framework/src/test/resources/webapps/testing/.gitkeep new file mode 100644 index 000000000000..e69de29bb2d1 From 97a67051f8cb0144aa7d5ce7453120757e47d736 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 13 May 2025 10:20:35 +0200 Subject: [PATCH 3/5] fix rat --- .../src/test/resources/webapps/testing/.gitkeep | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/hadoop-hdds/framework/src/test/resources/webapps/testing/.gitkeep b/hadoop-hdds/framework/src/test/resources/webapps/testing/.gitkeep index e69de29bb2d1..09697dce6e11 100644 --- a/hadoop-hdds/framework/src/test/resources/webapps/testing/.gitkeep +++ b/hadoop-hdds/framework/src/test/resources/webapps/testing/.gitkeep @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + From 9a2d1b4beaf8f484fe4be0c9b2aa52b430ce184e Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 13 May 2025 11:44:13 +0200 Subject: [PATCH 4/5] fix TestReconWithOzoneManager --- .../apache/hadoop/ozone/recon/TestReconWithOzoneManager.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java index adbbf7971c32..c3aa2db3f0dd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.recon.ReconConfigKeys; import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.utils.db.RDBStore; @@ -119,9 +120,7 @@ public static void init() throws Exception { cluster.getStorageContainerManager().exitSafeMode(); - InetSocketAddress address = - cluster.getReconServer().getHttpServer().getHttpAddress(); - String reconHTTPAddress = address.getHostName() + ":" + address.getPort(); + String reconHTTPAddress = conf.get(ReconConfigKeys.OZONE_RECON_HTTP_ADDRESS_KEY); taskStatusURL = "http://" + reconHTTPAddress + "/api/v1/task/status"; // initialize HTTPClient From f6712f8d0a05fe9163e213c46b025d84b9422067 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 13 May 2025 22:35:26 +0200 Subject: [PATCH 5/5] fix checkstyle --- .../org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java index c3aa2db3f0dd..f6aaab47d7ee 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java @@ -34,7 +34,6 @@ import static org.slf4j.event.Level.INFO; import java.io.IOException; -import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collections; import java.util.List;