From 0731eaf062b64b2672ec45c4edc23353e5202ee6 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Wed, 12 Mar 2025 02:00:15 +0530 Subject: [PATCH 01/17] [WIP] Migrate to jetty 12 with EE8 and bump java servlet to 4.0.1 - Also consume hbase-thridparty having the new jetty 12 ee8 module --- hbase-http/pom.xml | 2 +- .../hbase/http/AdminAuthorizedServlet.java | 2 +- .../apache/hadoop/hbase/http/HttpServer.java | 75 +++++++++++-------- .../hadoop/hbase/http/HttpServerUtil.java | 12 +-- .../apache/hadoop/hbase/http/InfoServer.java | 2 +- .../hbase/http/ProfileOutputServlet.java | 2 +- .../hadoop/hbase/http/TestHttpServer.java | 2 +- .../hbase/http/conf/TestConfServlet.java | 2 +- .../hbase/http/resource/JerseyResource.java | 2 +- .../apache/hadoop/hbase/MockHttpApiRule.java | 48 +++++++++--- .../hadoop/hbase/rest/NamespacesResource.java | 2 +- .../apache/hadoop/hbase/rest/RESTServer.java | 6 +- .../apache/hadoop/hbase/master/HMaster.java | 4 +- .../ensure-jars-have-correct-contents.sh | 3 + .../ensure-jars-have-correct-contents.sh | 3 + .../hadoop/hbase/thrift/ThriftServer.java | 4 +- pom.xml | 6 +- 17 files changed, 110 insertions(+), 67 deletions(-) diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml index 60288f495191..75cd85305630 100644 --- a/hbase-http/pom.xml +++ b/hbase-http/pom.xml @@ -78,7 +78,7 @@ org.apache.hbase.thirdparty - hbase-shaded-jetty + hbase-shaded-jetty-ee8 org.apache.hbase.thirdparty diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/AdminAuthorizedServlet.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/AdminAuthorizedServlet.java index 2ad09b5ae5c7..1ecd75532ce6 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/AdminAuthorizedServlet.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/AdminAuthorizedServlet.java @@ -24,7 +24,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.DefaultServlet; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.DefaultServlet; /** * General servlet which is admin-authorized. diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java index 6012b24ec543..fc4d474b0311 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java @@ -72,6 +72,12 @@ import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.DefaultServlet; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.FilterHolder; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.FilterMapping; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletContextHandler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.webapp.WebAppContext; import org.apache.hbase.thirdparty.org.eclipse.jetty.http.HttpVersion; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Handler; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConfiguration; @@ -84,18 +90,9 @@ import org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.ErrorHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.HandlerCollection; -import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.RequestLogHandler; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.gzip.GzipHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.DefaultServlet; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.FilterHolder; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.FilterMapping; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletContextHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder; -import org.apache.hbase.thirdparty.org.eclipse.jetty.util.MultiException; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ssl.SslContextFactory; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool; -import org.apache.hbase.thirdparty.org.eclipse.jetty.webapp.WebAppContext; import org.apache.hbase.thirdparty.org.glassfish.jersey.server.ResourceConfig; import org.apache.hbase.thirdparty.org.glassfish.jersey.servlet.ServletContainer; @@ -654,23 +651,21 @@ private void initializeWebServer(String name, String hostName, Configuration con Preconditions.checkNotNull(webAppContext); - HandlerCollection handlerCollection = new HandlerCollection(); + Handler.Sequence handlers = new Handler.Sequence(); ContextHandlerCollection contexts = new ContextHandlerCollection(); RequestLog requestLog = HttpRequestLog.getRequestLog(name); if (requestLog != null) { - RequestLogHandler requestLogHandler = new RequestLogHandler(); - requestLogHandler.setRequestLog(requestLog); - handlerCollection.addHandler(requestLogHandler); + webServer.setRequestLog(requestLog); } final String appDir = getWebAppsPath(name); - handlerCollection.addHandler(contexts); - handlerCollection.addHandler(webAppContext); + handlers.addHandler(contexts); + handlers.addHandler(webAppContext); - webServer.setHandler(handlerCollection); + webServer.setHandler(handlers); webAppContext.setAttribute(ADMINS_ACL, adminsAcl); @@ -715,8 +710,9 @@ private void initializeWebServer(String name, String hostName, Configuration con // Check if disable stack trace property is configured if (!conf.getBoolean(HTTP_UI_SHOW_STACKTRACE_KEY, true)) { // Disable stack traces for server errors in UI - webServer.setErrorHandler(new ErrorHandler()); - webServer.getErrorHandler().setShowStacks(false); + ErrorHandler errorHanlder = new ErrorHandler(); + errorHanlder.setShowStacks(false); + webServer.setErrorHandler(errorHanlder); // Disable stack traces for web app errors in UI webAppContext.getErrorHandler().setShowStacks(false); } @@ -841,7 +837,7 @@ private void configureAliasChecks(ServletContextHandler context, boolean shouldS if (context.getAliasChecks().stream().anyMatch(aliasCheckerClass::isInstance)) { LOG.debug("{} is already part of alias check list", aliasCheckerClass.getName()); } else { - context.addAliasCheck(new SymlinkAllowedResourceAliasChecker(context)); + context.addAliasCheck(new SymlinkAllowedResourceAliasChecker(context.getCoreContextHandler())); LOG.debug("{} added to the alias check list", aliasCheckerClass.getName()); } LOG.info("Serving aliases allowed for /logs context"); @@ -1258,14 +1254,14 @@ public void start() throws IOException { } catch (IOException ex) { LOG.info("HttpServer.start() threw a non Bind IOException", ex); throw ex; - } catch (MultiException ex) { + } catch (Exception ex) { LOG.info("HttpServer.start() threw a MultiException", ex); throw ex; } // Make sure there is no handler failures. - Handler[] handlers = webServer.getHandlers(); - for (int i = 0; i < handlers.length; i++) { - if (handlers[i].isFailed()) { + List handlers = webServer.getHandlers(); + for (Handler handler : handlers) { + if (handler.isFailed()) { throw new IOException("Problem in starting http server. Server handlers failed"); } } @@ -1335,7 +1331,7 @@ void openListeners() throws Exception { * stop the server */ public void stop() throws Exception { - MultiException exception = null; + Exception exception = null; for (ListenerInfo li : listeners) { if (!li.isManaged) { continue; @@ -1345,7 +1341,7 @@ public void stop() throws Exception { li.listener.close(); } catch (Exception e) { LOG.error("Error while stopping listener for webapp" + webAppContext.getDisplayName(), e); - exception = addMultiException(exception, e); + exception.addSuppressed(e); } } @@ -1356,28 +1352,41 @@ public void stop() throws Exception { } catch (Exception e) { LOG.error("Error while stopping web app context for webapp " + webAppContext.getDisplayName(), e); - exception = addMultiException(exception, e); + exception.addSuppressed(e); } try { webServer.stop(); } catch (Exception e) { LOG.error("Error while stopping web server for webapp " + webAppContext.getDisplayName(), e); - exception = addMultiException(exception, e); + exception.addSuppressed(e); } if (exception != null) { - exception.ifExceptionThrow(); + ifExceptionThrow(exception); } } - private MultiException addMultiException(MultiException exception, Exception e) { - if (exception == null) { - exception = new MultiException(); + /** + * Throw a {@link Throwable} as a checked {@link Exception} if it cannot be thrown as unchecked. + * @param throwable The {@link Throwable} to throw or null. + * @throws Error If the passed {@link Throwable} is an {@link Error}. + * @throws Exception Otherwise, if the passed {@link Throwable} is not null. + * + * NOTE: This method is copied from Jetty-9 + */ + public static void ifExceptionThrow(Throwable throwable) throws Error, Exception { + if (throwable == null) { + return; + } + if (throwable instanceof Error error) { + throw error; + } + if (throwable instanceof Exception exception) { + throw exception; } - exception.add(e); - return exception; + throw new RuntimeException(throwable); } public void join() throws InterruptedException { diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServerUtil.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServerUtil.java index ecfb32742fd1..0f1c9707d1c2 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServerUtil.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServerUtil.java @@ -22,11 +22,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hbase.thirdparty.org.eclipse.jetty.security.ConstraintMapping; -import org.apache.hbase.thirdparty.org.eclipse.jetty.security.ConstraintSecurityHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.FilterHolder; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletContextHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.util.security.Constraint; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.nested.ServletConstraint; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.security.ConstraintMapping; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.security.ConstraintSecurityHandler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.FilterHolder; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletContextHandler; /** * HttpServer utility. @@ -43,7 +43,7 @@ public final class HttpServerUtil { */ public static void constrainHttpMethods(ServletContextHandler ctxHandler, boolean allowOptionsMethod) { - Constraint c = new Constraint(); + ServletConstraint c = new ServletConstraint(); c.setAuthenticate(true); ConstraintMapping cmt = new ConstraintMapping(); diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java index ea73be808f07..5a09315ed774 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java @@ -29,7 +29,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.apache.hbase.thirdparty.com.google.common.net.HostAndPort; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder; /** * Create a Jetty embedded server to answer http requests. The primary goal is to serve up status diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileOutputServlet.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileOutputServlet.java index d92e7d009f68..12d0c4625044 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileOutputServlet.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileOutputServlet.java @@ -27,7 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.DefaultServlet; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.DefaultServlet; /** * Servlet to serve files generated by {@link ProfileServlet} diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java index ad9c9d3a0671..8354def7842e 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java @@ -507,7 +507,7 @@ public void testRequestQuoterWithNotNull() { @SuppressWarnings("unchecked") private static Map parse(String jsonString) { - return (Map) JSON.parse(jsonString); + return (Map) new JSON().fromJSON(jsonString); } @Test diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/conf/TestConfServlet.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/conf/TestConfServlet.java index fe202906e9c5..dc14be96d404 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/conf/TestConfServlet.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/conf/TestConfServlet.java @@ -73,7 +73,7 @@ public void testWriteJson() throws Exception { Set programSet = new HashSet<>(); programSet.add("programatically"); programSet.add("programmatically"); - Object parsed = JSON.parse(json); + Object parsed = new JSON().fromJSON(json); Object[] properties = ((Map) parsed).get("properties"); for (Object o : properties) { Map propertyInfo = (Map) o; diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/resource/JerseyResource.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/resource/JerseyResource.java index 89d71b403af7..4e97dcfae3fa 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/resource/JerseyResource.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/resource/JerseyResource.java @@ -54,7 +54,7 @@ public Response get(@PathParam(PATH) @DefaultValue("UNKNOWN_" + PATH) final Stri final Map m = new TreeMap<>(); m.put(PATH, path); m.put(OP, op); - final String js = JSON.toString(m); + final String js = new JSON().toJSON(m); return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java index 5817d071f02c..24763427c52f 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java @@ -28,17 +28,21 @@ import java.util.regex.Pattern; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Handler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.nested.AbstractHandler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Response; +import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Slf4jRequestLogWriter; +import org.apache.hbase.thirdparty.org.eclipse.jetty.util.Callback; import org.junit.rules.ExternalResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType; -import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Request; +import org.apache.hbase.thirdparty.org.eclipse.jetty.server.CustomRequestLog; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.nested.Request; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.RequestLog; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector; -import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Slf4jRequestLog; -import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.AbstractHandler; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.RegexSet; /** @@ -115,13 +119,12 @@ protected void after() { } private static RequestLog buildRequestLog() { - final Slf4jRequestLog requestLog = new Slf4jRequestLog(); - requestLog.setLoggerName(LOG.getName() + ".RequestLog"); - requestLog.setExtended(true); - return requestLog; + Slf4jRequestLogWriter writer = new Slf4jRequestLogWriter(); + writer.setLoggerName(LOG.getName() + ".RequestLog"); + return new CustomRequestLog(writer, CustomRequestLog.EXTENDED_NCSA_FORMAT); } - private static class MockHandler extends AbstractHandler { + private static class MockHandler extends Handler.Abstract { private final ReadWriteLock responseMappingLock = new ReentrantReadWriteLock(); private final Map> responseMapping = @@ -151,6 +154,32 @@ void clearRegistrations() { } @Override + public boolean handle(org.apache.hbase.thirdparty.org.eclipse.jetty.server.Request request, Response response, Callback callback) throws Exception { + // TODO: TestRESTApiClusterManager is failing because of this line?? + String target = request.getHttpURI().asString(); + responseMappingLock.readLock().lock(); + try { + if (!regexSet.matches(target)) { + response.setStatus(HttpServletResponse.SC_NOT_FOUND); + callback.succeeded(); + return true; + } + responseMapping.entrySet().stream() + .filter(e -> Pattern.matches(e.getKey(), target)) + .findAny() + .map(Map.Entry::getValue) + .orElseThrow(() -> noMatchFound(target)) + .accept(target, (HttpServletResponse) response); + callback.succeeded(); + } catch (Exception e) { + callback.failed(e); + } finally { + responseMappingLock.readLock().unlock(); + } + return true; + } + + //@Override public void handle(final String target, final Request baseRequest, final HttpServletRequest request, final HttpServletResponse response) { responseMappingLock.readLock().lock(); @@ -168,8 +197,7 @@ public void handle(final String target, final Request baseRequest, } private static RuntimeException noMatchFound(final String target) { - return new RuntimeException( - String.format("Target path '%s' matches no registered regex.", target)); + return new RuntimeException(String.format("Target path '%s' matches no registered regex.", target)); } } } diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesResource.java index aeccda24f197..169a652bbed5 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesResource.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesResource.java @@ -71,7 +71,7 @@ public Response get(final @Context ServletContext context, final @Context UriInf return Response.ok(rowModel).build(); } catch (IOException e) { servlet.getMetrics().incrementFailedGetRequests(1); - throw new RuntimeException("Cannot retrieve list of namespaces."); + throw new RuntimeException("Cannot retrieve list of namespaces.", e); } } diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java index 760f2ca8b41c..ba3fc9af33bd 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java @@ -57,6 +57,9 @@ import org.apache.hbase.thirdparty.org.apache.commons.cli.Options; import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException; import org.apache.hbase.thirdparty.org.apache.commons.cli.PosixParser; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.FilterHolder; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletContextHandler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder; import org.apache.hbase.thirdparty.org.eclipse.jetty.http.HttpVersion; import org.apache.hbase.thirdparty.org.eclipse.jetty.jmx.MBeanContainer; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConfiguration; @@ -65,9 +68,6 @@ import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.SslConnectionFactory; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.FilterHolder; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletContextHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ssl.SslContextFactory; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool; import org.apache.hbase.thirdparty.org.glassfish.jersey.server.ResourceConfig; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 19f58ebe6ad0..317f571ba8ac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -287,10 +287,10 @@ import org.apache.hbase.thirdparty.com.google.gson.JsonParseException; import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors; import org.apache.hbase.thirdparty.com.google.protobuf.Service; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.webapp.WebAppContext; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder; -import org.apache.hbase.thirdparty.org.eclipse.jetty.webapp.WebAppContext; import org.apache.hbase.thirdparty.org.glassfish.jersey.server.ResourceConfig; import org.apache.hbase.thirdparty.org.glassfish.jersey.servlet.ServletContainer; diff --git a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index 1d1350712b12..17fa4bb2070b 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -98,6 +98,9 @@ allowed_expr+="|^about.html$" allowed_expr+="|^jetty-dir.css$" # Coming from Guava, see https://github.com/google/guava/commit/2cc8c5eddb587db3ac12dacdd5563e79a4681ec4 allowed_expr+="|^org/jspecify/$|^org/jspecify/annotations/$|^org/jspecify/annotations/.*\.class$" +# Required by jetty 12 on ee8 +allowed_expr="(|^javax/$)" + if [ -n "${allow_hadoop}" ]; then # * classes in packages that start with org.apache.hadoop, which by diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index 1d1350712b12..17fa4bb2070b 100644 --- a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -98,6 +98,9 @@ allowed_expr+="|^about.html$" allowed_expr+="|^jetty-dir.css$" # Coming from Guava, see https://github.com/google/guava/commit/2cc8c5eddb587db3ac12dacdd5563e79a4681ec4 allowed_expr+="|^org/jspecify/$|^org/jspecify/annotations/$|^org/jspecify/annotations/.*\.class$" +# Required by jetty 12 on ee8 +allowed_expr="(|^javax/$)" + if [ -n "${allow_hadoop}" ]; then # * classes in packages that start with org.apache.hadoop, which by diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java index 7f2d37440297..a0370aab0fb1 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java @@ -144,6 +144,8 @@ import org.apache.hbase.thirdparty.org.apache.commons.cli.DefaultParser; import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter; import org.apache.hbase.thirdparty.org.apache.commons.cli.Options; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletContextHandler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder; import org.apache.hbase.thirdparty.org.eclipse.jetty.http.HttpVersion; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConfiguration; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConnectionFactory; @@ -151,8 +153,6 @@ import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.SslConnectionFactory; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletContextHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ssl.SslContextFactory; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool; diff --git a/pom.xml b/pom.xml index 30c63fca3819..121467214dc4 100644 --- a/pom.xml +++ b/pom.xml @@ -893,7 +893,7 @@ 2.19.2 2.19.2 2.3.1 - 3.1.0 + 4.0.1 2.1.1 9.0.104 9.4.12.1 @@ -975,7 +975,7 @@ - 9.0.93 + ${tomcat.jasper.version} 0.8.8 @@ -1847,7 +1847,7 @@ org.apache.hbase.thirdparty - hbase-shaded-jetty + hbase-shaded-jetty-ee8 ${hbase-thirdparty.version} From 84b40cb834fceaebe63773f75d354b52b8dea7f6 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Thu, 27 Mar 2025 17:33:55 +0530 Subject: [PATCH 02/17] Make changes to fix UTs, fix /logs page not loading issue etc. --- .../apache/hadoop/hbase/http/HttpServer.java | 14 +- .../hadoop/hbase/http/log/LogLevel.java | 7 +- .../http/log/LogLevelExceptionUtils.java | 103 ++++ .../hadoop/hbase/http/TestHttpServer.java | 3 + .../hadoop/hbase/http/log/TestLogLevel.java | 5 +- .../apache/hadoop/hbase/MockHttpApiRule.java | 83 ++- .../apache/hadoop/hbase/rest/RESTServer.java | 18 + .../hbase/rest/TestGetAndPutResource.java | 2 + .../hbase/rest/TestSecureRESTServer.java | 4 +- .../hbase/rest/client/TestRemoteTable.java | 8 + .../org/eclipse/jetty/webapp/webdefault.xml | 550 ------------------ 11 files changed, 187 insertions(+), 610 deletions(-) create mode 100644 hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java delete mode 100644 hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java index fc4d474b0311..73c174635ae1 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java @@ -790,7 +790,7 @@ private static FilterInitializer[] getFilterInitializers(Configuration conf) { * @param appDir The application directory */ protected void addDefaultApps(ContextHandlerCollection parent, final String appDir, - Configuration conf) { + Configuration conf) throws IOException { // set up the context for "/logs/" if "hadoop.log.dir" property is defined. String logDir = this.logDir; if (logDir == null) { @@ -799,7 +799,9 @@ protected void addDefaultApps(ContextHandlerCollection parent, final String appD if (logDir != null) { ServletContextHandler logContext = new ServletContextHandler(parent, "/logs"); logContext.addServlet(AdminAuthorizedServlet.class, "/*"); - logContext.setResourceBase(logDir); + // We are doing this as otherwise jetty 12 is not handling /dir0/dir1/../dir2 + String logDirCanonical = Paths.get(logDir).toFile().getCanonicalPath(); + logContext.setResourceBase(logDirCanonical); logContext.setDisplayName("logs"); configureAliasChecks(logContext, conf.getBoolean(ServerConfigurationKeys.HBASE_JETTY_LOGS_SERVE_ALIASES, @@ -837,7 +839,8 @@ private void configureAliasChecks(ServletContextHandler context, boolean shouldS if (context.getAliasChecks().stream().anyMatch(aliasCheckerClass::isInstance)) { LOG.debug("{} is already part of alias check list", aliasCheckerClass.getName()); } else { - context.addAliasCheck(new SymlinkAllowedResourceAliasChecker(context.getCoreContextHandler())); + context + .addAliasCheck(new SymlinkAllowedResourceAliasChecker(context.getCoreContextHandler())); LOG.debug("{} added to the alias check list", aliasCheckerClass.getName()); } LOG.info("Serving aliases allowed for /logs context"); @@ -1370,11 +1373,12 @@ public void stop() throws Exception { /** * Throw a {@link Throwable} as a checked {@link Exception} if it cannot be thrown as unchecked. + * + * NOTE: This method is copied from Jetty-9 + * * @param throwable The {@link Throwable} to throw or null. * @throws Error If the passed {@link Throwable} is an {@link Error}. * @throws Exception Otherwise, if the passed {@link Throwable} is not null. - * - * NOTE: This method is copied from Jetty-9 */ public static void ifExceptionThrow(Throwable throwable) throws Error, Exception { if (throwable == null) { diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java index a9e8fa7cbe16..5db4ee1cac0a 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java @@ -41,7 +41,6 @@ import org.apache.hadoop.security.authentication.client.AuthenticatedURL; import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; import org.apache.hadoop.security.ssl.SSLFactory; -import org.apache.hadoop.util.HttpExceptionUtils; import org.apache.hadoop.util.ServletUtil; import org.apache.hadoop.util.Tool; import org.apache.yetus.audience.InterfaceAudience; @@ -267,7 +266,11 @@ private void process(String urlString) throws Exception { HttpURLConnection connection = connect(url); - HttpExceptionUtils.validateResponse(connection, 200); + // We implement the validateResponse method inside hbase to handle for HTML response. + // as with Jetty 12: getResponseMessage returns "Precondition Failed" vs + // "Modification of logger protected.org.apache.hadoop.hbase.http.log.TestLogLevel is + // disallowed in configuration" in Jetty 9 + LogLevelExceptionUtils.validateResponse(connection, 200); // read from the servlet diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java new file mode 100644 index 000000000000..d6b7f519d6b2 --- /dev/null +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.http.log; + +import java.io.IOException; +import java.io.InputStream; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; +import java.net.HttpURLConnection; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * HTTP utility class to help propagate server side exception in log level servlet to the client + * over HTTP (HTML payload) + * It parses HTTP client connections and recreates the exception. + */ +@InterfaceAudience.Private public class LogLevelExceptionUtils { + + private static void throwEx(Throwable ex) { + LogLevelExceptionUtils.throwException(ex); + } + + @SuppressWarnings("unchecked") + private static void throwException(Throwable ex) throws E { + throw (E) ex; + } + + /** + * Validates the status of an HttpURLConnection against an + * expected HTTP status code. If the current status code is not the expected + * one it throws an exception with a detail message using Server side error + * messages if available. + *

+ * NOTE: this method will throw the deserialized exception even if not + * declared in the throws of the method signature. + * + * @param conn the HttpURLConnection. + * @param expectedStatus the expected HTTP status code. + * @throws IOException thrown if the current status code does not match the + * expected one. + * NOTE: This is an adapted version of the original method in HttpServerUtil.java from Hadoop + * but we handle for HTML response. + */ + @SuppressWarnings("unchecked") public static void validateResponse(HttpURLConnection conn, + int expectedStatus) throws IOException { + if (conn.getResponseCode() != expectedStatus) { + Exception toThrow = null; + + try (InputStream es = conn.getErrorStream()) { + if (es != null) { + final String errorAsHtml = + new BufferedReader(new InputStreamReader(es, StandardCharsets.UTF_8)).lines() + .collect(Collectors.joining("\n")); + + final String status = extractValue(errorAsHtml, "STATUS:(\\d+)"); + final String message = extractValue(errorAsHtml, "MESSAGE:([^<]+)"); + final String uri = extractValue(errorAsHtml, "URI:([^<]+)"); + final String exception = extractValue(errorAsHtml, "([^<]+)"); + + toThrow = new IOException( + String.format("HTTP status [%s], message [%s], URL [%s], exception [%s]", status, + message, uri, exception)); + } + } catch (Exception ex) { + toThrow = new IOException( + String.format("HTTP status [%d], message [%s], URL [%s], exception [%s]", + conn.getResponseCode(), conn.getResponseMessage(), conn.getURL(), ex), ex); + } + if (toThrow != null) { + throwEx(toThrow); + } + } + } + + private static String extractValue(String html, String regex) { + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(html); + if (matcher.find()) { + return matcher.group(1); + } + return null; + } + +} diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java index 8354def7842e..724663928271 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java @@ -615,6 +615,9 @@ private HttpServer checkBindAddress(String host, int port, boolean findPort) thr ServerConnector listener = server.getServerConnectors().get(0); assertEquals(port, listener.getPort()); + // We are doing this as otherwise testBindAddress fails, not sure how we were even starting + // server in jetty 9 without this call + server.start(); // verify hostname is what was given server.openListeners(); assertEquals(host, server.getConnectorAddress(0).getHostName()); diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java index bc45a6295551..04452f613bae 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java @@ -235,7 +235,10 @@ private boolean validateCommand(String[] args) { * @throws Exception if unable to create or start a Jetty server */ private HttpServer createServer(String protocol, boolean isSpnego) throws Exception { - HttpServer.Builder builder = new HttpServer.Builder().setName("..") + // Changed to "" as ".." moves it a steps back in path because the path is relative to the + // current working directory. throws "java.lang.IllegalArgumentException: Base Resource is not + // valid: hbase-http/target/test-classes/static" as it is not able to find the static folder. + HttpServer.Builder builder = new HttpServer.Builder().setName("") .addEndpoint(new URI(protocol + "://localhost:0")).setFindPort(true).setConf(serverConf); if (isSpnego) { // Set up server Kerberos credentials. diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java index 24763427c52f..be50b3137256 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java @@ -17,32 +17,31 @@ */ package org.apache.hadoop.hbase; -import java.io.IOException; -import java.io.PrintWriter; import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.BiConsumer; import java.util.regex.Pattern; -import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Handler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.nested.AbstractHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Response; -import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Slf4jRequestLogWriter; -import org.apache.hbase.thirdparty.org.eclipse.jetty.util.Callback; import org.junit.rules.ExternalResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.javax.ws.rs.core.HttpHeaders; import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.CustomRequestLog; -import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.nested.Request; +import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Handler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Request; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.RequestLog; +import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Response; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector; +import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Slf4jRequestLogWriter; +import org.apache.hbase.thirdparty.org.eclipse.jetty.util.Callback; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.RegexSet; /** @@ -59,7 +58,7 @@ public class MockHttpApiRule extends ExternalResource { * Register a callback handler for the specified path target. */ public MockHttpApiRule addRegistration(final String pathRegex, - final BiConsumer responder) { + final BiConsumer responder) { handler.register(pathRegex, responder); return this; } @@ -69,16 +68,21 @@ public MockHttpApiRule addRegistration(final String pathRegex, */ public MockHttpApiRule registerOk(final String pathRegex, final String responseBody) { return addRegistration(pathRegex, (target, resp) -> { - try { - resp.setStatus(HttpServletResponse.SC_OK); - resp.setCharacterEncoding("UTF-8"); - resp.setContentType(MediaType.APPLICATION_JSON_TYPE.toString()); - final PrintWriter writer = resp.getWriter(); - writer.write(responseBody); - writer.flush(); - } catch (IOException e) { - throw new RuntimeException(e); - } + resp.setStatus(HttpServletResponse.SC_OK); + resp.getHeaders().put(HttpHeaders.CONTENT_ENCODING, "UTF-8"); + resp.getHeaders().put(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_TYPE.toString()); + ByteBuffer content = ByteBuffer.wrap(responseBody.getBytes(StandardCharsets.UTF_8)); + resp.write(true, content, new Callback() { + @Override + public void succeeded() { + // nothing to do + } + + @Override + public void failed(Throwable x) { + throw new RuntimeException(x); + } + }); }); } @@ -127,11 +131,10 @@ private static RequestLog buildRequestLog() { private static class MockHandler extends Handler.Abstract { private final ReadWriteLock responseMappingLock = new ReentrantReadWriteLock(); - private final Map> responseMapping = - new HashMap<>(); + private final Map> responseMapping = new HashMap<>(); private final RegexSet regexSet = new RegexSet(); - void register(final String pathRegex, final BiConsumer responder) { + void register(final String pathRegex, final BiConsumer responder) { LOG.debug("Registering responder to '{}'", pathRegex); responseMappingLock.writeLock().lock(); try { @@ -154,9 +157,8 @@ void clearRegistrations() { } @Override - public boolean handle(org.apache.hbase.thirdparty.org.eclipse.jetty.server.Request request, Response response, Callback callback) throws Exception { - // TODO: TestRESTApiClusterManager is failing because of this line?? - String target = request.getHttpURI().asString(); + public boolean handle(Request request, Response response, Callback callback) throws Exception { + String target = request.getHttpURI().getPath(); responseMappingLock.readLock().lock(); try { if (!regexSet.matches(target)) { @@ -164,12 +166,9 @@ public boolean handle(org.apache.hbase.thirdparty.org.eclipse.jetty.server.Reque callback.succeeded(); return true; } - responseMapping.entrySet().stream() - .filter(e -> Pattern.matches(e.getKey(), target)) - .findAny() - .map(Map.Entry::getValue) - .orElseThrow(() -> noMatchFound(target)) - .accept(target, (HttpServletResponse) response); + responseMapping.entrySet().stream().filter(e -> Pattern.matches(e.getKey(), target)) + .findAny().map(Map.Entry::getValue).orElseThrow(() -> noMatchFound(target)) + .accept(target, response); callback.succeeded(); } catch (Exception e) { callback.failed(e); @@ -179,25 +178,9 @@ public boolean handle(org.apache.hbase.thirdparty.org.eclipse.jetty.server.Reque return true; } - //@Override - public void handle(final String target, final Request baseRequest, - final HttpServletRequest request, final HttpServletResponse response) { - responseMappingLock.readLock().lock(); - try { - if (!regexSet.matches(target)) { - response.setStatus(HttpServletResponse.SC_NOT_FOUND); - return; - } - responseMapping.entrySet().stream().filter(e -> Pattern.matches(e.getKey(), target)) - .findAny().map(Map.Entry::getValue).orElseThrow(() -> noMatchFound(target)) - .accept(target, response); - } finally { - responseMappingLock.readLock().unlock(); - } - } - private static RuntimeException noMatchFound(final String target) { - return new RuntimeException(String.format("Target path '%s' matches no registered regex.", target)); + return new RuntimeException( + String.format("Target path '%s' matches no registered regex.", target)); } } } diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java index ba3fc9af33bd..e672370b2171 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java @@ -23,8 +23,10 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import javax.servlet.DispatcherType; import org.apache.commons.lang3.ArrayUtils; @@ -61,6 +63,7 @@ import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletContextHandler; import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder; import org.apache.hbase.thirdparty.org.eclipse.jetty.http.HttpVersion; +import org.apache.hbase.thirdparty.org.eclipse.jetty.http.UriCompliance; import org.apache.hbase.thirdparty.org.eclipse.jetty.jmx.MBeanContainer; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConfiguration; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConnectionFactory; @@ -286,6 +289,7 @@ public synchronized void run() throws Exception { httpConfig.setResponseHeaderSize(DEFAULT_HTTP_MAX_HEADER_SIZE); httpConfig.setSendServerVersion(false); httpConfig.setSendDateHeader(false); + setUriComplianceRules(httpConfig); ServerConnector serverConnector; boolean isSecure = false; @@ -400,6 +404,20 @@ public synchronized void run() throws Exception { server.start(); } + private static void setUriComplianceRules(HttpConfiguration httpConfig) { + // In Jetty 12, ambiguous path separators, suspicious path characters, and ambiguous empty + // segments are considered violations of the URI specification and hence are not allowed. + // Refer to https://github.com/jetty/jetty.project/issues/11890#issuecomment-2156449534 + // We must set a URI compliance to allow for this violation so that client + // requests are not automatically rejected. We have tests which rely on this behavior. + // TODO Discuss Should we set below to UriCompliance.LEGACY instead of cherry-picking? + Set complianceViolationSet = new HashSet<>(); + complianceViolationSet.add(UriCompliance.Violation.AMBIGUOUS_PATH_SEPARATOR); + complianceViolationSet.add(UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS); + complianceViolationSet.add(UriCompliance.Violation.AMBIGUOUS_EMPTY_SEGMENT); + httpConfig.setUriCompliance(UriCompliance.from(complianceViolationSet)); + } + private static String getHostName(Configuration conf) throws UnknownHostException { return Strings.domainNamePointerToHostName(DNS.getDefaultHost( conf.get(REST_DNS_INTERFACE, "default"), conf.get(REST_DNS_NAMESERVER, "default"))); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java index b20baea9df8c..2751354f7418 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java @@ -321,6 +321,8 @@ public void testLatestCellGetJSON() throws IOException { @Test public void testURLEncodedKey() throws IOException, JAXBException { + // Requires UriCompliance.Violation.AMBIGUOUS_PATH_SEPARATOR + // Otherwise fails with "400: Ambiguous URI path separator" String urlKey = "http://example.com/foo"; StringBuilder path = new StringBuilder(); path.append('/'); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecureRESTServer.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecureRESTServer.java index 4ad065cbdc43..a7125417b46a 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecureRESTServer.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecureRESTServer.java @@ -54,7 +54,7 @@ import org.apache.hadoop.hbase.security.access.Permission.Action; import org.apache.hadoop.hbase.security.token.TokenProvider; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SecurityTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -101,7 +101,7 @@ * Test class for SPNEGO authentication on the HttpServer. Uses Kerby's MiniKDC and Apache * HttpComponents to verify that a simple Servlet is reachable via SPNEGO and unreachable w/o. */ -@Category({ MiscTests.class, MediumTests.class }) +@Category({ SecurityTests.class, MediumTests.class }) public class TestSecureRESTServer { @ClassRule diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java index fcbe7c7ca02e..c33a42e1473d 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java @@ -164,6 +164,8 @@ public void testGetTableDescriptor() throws IOException { @Test public void testGet() throws IOException { + // Requires UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS + // Otherwise fails with "400: Suspicious Path Character" Get get = new Get(ROW_1); Result result = remoteTable.get(get); byte[] value1 = result.getValue(COLUMN_1, QUALIFIER_1); @@ -303,6 +305,8 @@ public void testMultiGet() throws Exception { @Test public void testPut() throws IOException { + // Requires UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS + // Otherwise fails with "400: Suspicious Path Character" Put put = new Put(ROW_3); put.addColumn(COLUMN_1, QUALIFIER_1, VALUE_1); remoteTable.put(put); @@ -348,6 +352,8 @@ public void testPut() throws IOException { @Test public void testDelete() throws IOException { + // Requires UriCompliance.Violation.AMBIGUOUS_EMPTY_SEGMENT + // Otherwise fails with "400: Ambiguous URI empty segment" Put put = new Put(ROW_3); put.addColumn(COLUMN_1, QUALIFIER_1, VALUE_1); put.addColumn(COLUMN_2, QUALIFIER_2, VALUE_2); @@ -493,6 +499,8 @@ public void testScanner() throws IOException { @Test public void testCheckAndDelete() throws IOException { + // Requires UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS + // Otherwise fails with "400: Suspicious Path Character" Get get = new Get(ROW_1); Result result = remoteTable.get(get); byte[] value1 = result.getValue(COLUMN_1, QUALIFIER_1); diff --git a/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml b/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml deleted file mode 100644 index 8f10b517eb55..000000000000 --- a/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml +++ /dev/null @@ -1,550 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - Default web.xml file. - This file is applied to a Web application before it's own WEB_INF/web.xml file - - - - - - - - org.apache.hadoop.hbase.shaded.org.eclipse.jetty.servlet.listener.ELContextCleaner - - - - - - - - org.apache.hadoop.hbase.shaded.org.eclipse.jetty.servlet.listener.IntrospectorCleaner - - - - - - - - - - - - - - - - - default - org.apache.hadoop.hbase.shaded.org.eclipse.jetty.servlet.DefaultServlet - - aliases - false - - - acceptRanges - true - - - dirAllowed - true - - - welcomeServlets - false - - - redirectWelcome - false - - - maxCacheSize - 256000000 - - - maxCachedFileSize - 200000000 - - - maxCachedFiles - 2048 - - - gzip - false - - - etags - false - - - useFileMappedBuffer - true - - - - 0 - - - - default - / - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - jsp - org.apache.hadoop.hbase.shaded.org.eclipse.jetty.jsp.JettyJspServlet - - logVerbosityLevel - DEBUG - - - fork - false - - - xpoweredBy - false - - - compilerTargetVM - 1.7 - - - compilerSourceVM - 1.7 - - - 0 - - - - jsp - *.jsp - *.jspf - *.jspx - *.xsp - *.JSP - *.JSPF - *.JSPX - *.XSP - - - - - - - - 30 - - - - - - - - - - - - - - - index.html - index.htm - index.jsp - - - - - - - - ar - ISO-8859-6 - - - be - ISO-8859-5 - - - bg - ISO-8859-5 - - - ca - ISO-8859-1 - - - cs - ISO-8859-2 - - - da - ISO-8859-1 - - - de - ISO-8859-1 - - - el - ISO-8859-7 - - - en - ISO-8859-1 - - - es - ISO-8859-1 - - - et - ISO-8859-1 - - - fi - ISO-8859-1 - - - fr - ISO-8859-1 - - - hr - ISO-8859-2 - - - hu - ISO-8859-2 - - - is - ISO-8859-1 - - - it - ISO-8859-1 - - - iw - ISO-8859-8 - - - ja - Shift_JIS - - - ko - EUC-KR - - - lt - ISO-8859-2 - - - lv - ISO-8859-2 - - - mk - ISO-8859-5 - - - nl - ISO-8859-1 - - - no - ISO-8859-1 - - - pl - ISO-8859-2 - - - pt - ISO-8859-1 - - - ro - ISO-8859-2 - - - ru - ISO-8859-5 - - - sh - ISO-8859-5 - - - sk - ISO-8859-2 - - - sl - ISO-8859-2 - - - sq - ISO-8859-2 - - - sr - ISO-8859-5 - - - sv - ISO-8859-1 - - - tr - ISO-8859-9 - - - uk - ISO-8859-5 - - - zh - GB2312 - - - zh_TW - Big5 - - - - - - - - - Disable TRACE - / - TRACE - - - - - - Enable everything but TRACE - / - TRACE - - - - From 840c0db6534e83e50d1d96118855e41f36e28432 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Thu, 27 Mar 2025 18:04:02 +0530 Subject: [PATCH 03/17] Fix spotless --- .../apache/hadoop/hbase/http/HttpServer.java | 5 +-- .../http/log/LogLevelExceptionUtils.java | 40 +++++++++---------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java index 73c174635ae1..276a79311deb 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java @@ -1373,9 +1373,8 @@ public void stop() throws Exception { /** * Throw a {@link Throwable} as a checked {@link Exception} if it cannot be thrown as unchecked. - * - * NOTE: This method is copied from Jetty-9 - * + *

+ * NOTE: This method is copied from Jetty-9 * @param throwable The {@link Throwable} to throw or null. * @throws Error If the passed {@link Throwable} is an {@link Error}. * @throws Exception Otherwise, if the passed {@link Throwable} is not null. diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java index d6b7f519d6b2..f19085943705 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java @@ -17,26 +17,26 @@ */ package org.apache.hadoop.hbase.http.log; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; -import java.io.BufferedReader; import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.stream.Collectors; import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.yetus.audience.InterfaceAudience; /** * HTTP utility class to help propagate server side exception in log level servlet to the client - * over HTTP (HTML payload) - * It parses HTTP client connections and recreates the exception. + * over HTTP (HTML payload) It parses HTTP client connections and recreates the exception. */ -@InterfaceAudience.Private public class LogLevelExceptionUtils { +@InterfaceAudience.Private +public class LogLevelExceptionUtils { private static void throwEx(Throwable ex) { - LogLevelExceptionUtils.throwException(ex); + LogLevelExceptionUtils. throwException(ex); } @SuppressWarnings("unchecked") @@ -45,23 +45,19 @@ private static void throwException(Throwable ex) throws E } /** - * Validates the status of an HttpURLConnection against an - * expected HTTP status code. If the current status code is not the expected - * one it throws an exception with a detail message using Server side error - * messages if available. + * Validates the status of an HttpURLConnection against an expected HTTP status code. + * If the current status code is not the expected one it throws an exception with a detail message + * using Server side error messages if available. *

- * NOTE: this method will throw the deserialized exception even if not - * declared in the throws of the method signature. - * + * NOTE: This is an adapted version of the original method in HttpServerUtil.java of Hadoop, + * but we handle for HTML response. * @param conn the HttpURLConnection. * @param expectedStatus the expected HTTP status code. - * @throws IOException thrown if the current status code does not match the - * expected one. - * NOTE: This is an adapted version of the original method in HttpServerUtil.java from Hadoop - * but we handle for HTML response. + * @throws IOException thrown if the current status code does not match the expected one. */ - @SuppressWarnings("unchecked") public static void validateResponse(HttpURLConnection conn, - int expectedStatus) throws IOException { + @SuppressWarnings("unchecked") + public static void validateResponse(HttpURLConnection conn, int expectedStatus) + throws IOException { if (conn.getResponseCode() != expectedStatus) { Exception toThrow = null; @@ -81,8 +77,8 @@ private static void throwException(Throwable ex) throws E message, uri, exception)); } } catch (Exception ex) { - toThrow = new IOException( - String.format("HTTP status [%d], message [%s], URL [%s], exception [%s]", + toThrow = + new IOException(String.format("HTTP status [%d], message [%s], URL [%s], exception [%s]", conn.getResponseCode(), conn.getResponseMessage(), conn.getURL(), ex), ex); } if (toThrow != null) { From c1aa3fc61e07b170414de3199769cd5f9810fb2f Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Thu, 27 Mar 2025 20:13:35 +0530 Subject: [PATCH 04/17] Make changes to test EE8 + EE9 in single module --- hbase-http/pom.xml | 2 +- .../java/org/apache/hadoop/hbase/rest/NamespacesResource.java | 2 +- .../org/apache/hadoop/hbase/rest/TestSecureRESTServer.java | 4 ++-- .../src/test/resources/ensure-jars-have-correct-contents.sh | 3 ++- .../src/test/resources/ensure-jars-have-correct-contents.sh | 3 ++- pom.xml | 2 +- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml index 75cd85305630..6a4c79aa9b4e 100644 --- a/hbase-http/pom.xml +++ b/hbase-http/pom.xml @@ -78,7 +78,7 @@ org.apache.hbase.thirdparty - hbase-shaded-jetty-ee8 + hbase-shaded-jetty-12-plus org.apache.hbase.thirdparty diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesResource.java index 169a652bbed5..aeccda24f197 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesResource.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesResource.java @@ -71,7 +71,7 @@ public Response get(final @Context ServletContext context, final @Context UriInf return Response.ok(rowModel).build(); } catch (IOException e) { servlet.getMetrics().incrementFailedGetRequests(1); - throw new RuntimeException("Cannot retrieve list of namespaces.", e); + throw new RuntimeException("Cannot retrieve list of namespaces."); } } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecureRESTServer.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecureRESTServer.java index a7125417b46a..4ad065cbdc43 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecureRESTServer.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecureRESTServer.java @@ -54,7 +54,7 @@ import org.apache.hadoop.hbase.security.access.Permission.Action; import org.apache.hadoop.hbase.security.token.TokenProvider; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.testclassification.SecurityTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -101,7 +101,7 @@ * Test class for SPNEGO authentication on the HttpServer. Uses Kerby's MiniKDC and Apache * HttpComponents to verify that a simple Servlet is reachable via SPNEGO and unreachable w/o. */ -@Category({ SecurityTests.class, MediumTests.class }) +@Category({ MiscTests.class, MediumTests.class }) public class TestSecureRESTServer { @ClassRule diff --git a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index 17fa4bb2070b..c237ba9af7ed 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -100,7 +100,8 @@ allowed_expr+="|^jetty-dir.css$" allowed_expr+="|^org/jspecify/$|^org/jspecify/annotations/$|^org/jspecify/annotations/.*\.class$" # Required by jetty 12 on ee8 allowed_expr="(|^javax/$)" - +# Required by jetty 12 on ee9 +allowed_expr="(|^jakarta/$)" if [ -n "${allow_hadoop}" ]; then # * classes in packages that start with org.apache.hadoop, which by diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index 17fa4bb2070b..c237ba9af7ed 100644 --- a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -100,7 +100,8 @@ allowed_expr+="|^jetty-dir.css$" allowed_expr+="|^org/jspecify/$|^org/jspecify/annotations/$|^org/jspecify/annotations/.*\.class$" # Required by jetty 12 on ee8 allowed_expr="(|^javax/$)" - +# Required by jetty 12 on ee9 +allowed_expr="(|^jakarta/$)" if [ -n "${allow_hadoop}" ]; then # * classes in packages that start with org.apache.hadoop, which by diff --git a/pom.xml b/pom.xml index 121467214dc4..3e9bccdf6284 100644 --- a/pom.xml +++ b/pom.xml @@ -1847,7 +1847,7 @@ org.apache.hbase.thirdparty - hbase-shaded-jetty-ee8 + hbase-shaded-jetty-12-plus ${hbase-thirdparty.version} From be3457c0dfccaa63195c77bf1a0516148139f9e8 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Fri, 28 Mar 2025 21:53:08 +0530 Subject: [PATCH 05/17] Update as per thirdparty changes post review --- hbase-http/pom.xml | 6 +++++- pom.xml | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml index 6a4c79aa9b4e..98f7c4e1a06f 100644 --- a/hbase-http/pom.xml +++ b/hbase-http/pom.xml @@ -78,7 +78,11 @@ org.apache.hbase.thirdparty - hbase-shaded-jetty-12-plus + hbase-shaded-jetty-12-plus-core + + + org.apache.hbase.thirdparty + hbase-shaded-jetty-12-plus-ee8 org.apache.hbase.thirdparty diff --git a/pom.xml b/pom.xml index 3e9bccdf6284..056abf7b836e 100644 --- a/pom.xml +++ b/pom.xml @@ -1847,7 +1847,12 @@ org.apache.hbase.thirdparty - hbase-shaded-jetty-12-plus + hbase-shaded-jetty-12-plus-core + ${hbase-thirdparty.version} + + + org.apache.hbase.thirdparty + hbase-shaded-jetty-12-plus-ee8 ${hbase-thirdparty.version} From b98a44a348ec9c693930b26da062d842c2ba447a Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Sat, 29 Mar 2025 00:06:45 +0530 Subject: [PATCH 06/17] Fix type and use ExceptionUtil.MultiException and revert to old code --- .../apache/hadoop/hbase/http/HttpServer.java | 40 +++++++------------ 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java index 276a79311deb..0fa09ce6c570 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java @@ -80,6 +80,7 @@ import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.webapp.WebAppContext; import org.apache.hbase.thirdparty.org.eclipse.jetty.http.HttpVersion; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Handler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ExceptionUtil; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConfiguration; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConnectionFactory; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.RequestLog; @@ -710,9 +711,9 @@ private void initializeWebServer(String name, String hostName, Configuration con // Check if disable stack trace property is configured if (!conf.getBoolean(HTTP_UI_SHOW_STACKTRACE_KEY, true)) { // Disable stack traces for server errors in UI - ErrorHandler errorHanlder = new ErrorHandler(); - errorHanlder.setShowStacks(false); - webServer.setErrorHandler(errorHanlder); + ErrorHandler errorHandler = new ErrorHandler(); + errorHandler.setShowStacks(false); + webServer.setErrorHandler(errorHandler); // Disable stack traces for web app errors in UI webAppContext.getErrorHandler().setShowStacks(false); } @@ -1334,7 +1335,7 @@ void openListeners() throws Exception { * stop the server */ public void stop() throws Exception { - Exception exception = null; + ExceptionUtil.MultiException exception = null; for (ListenerInfo li : listeners) { if (!li.isManaged) { continue; @@ -1344,7 +1345,7 @@ public void stop() throws Exception { li.listener.close(); } catch (Exception e) { LOG.error("Error while stopping listener for webapp" + webAppContext.getDisplayName(), e); - exception.addSuppressed(e); + exception = addMultiException(exception, e); } } @@ -1355,41 +1356,28 @@ public void stop() throws Exception { } catch (Exception e) { LOG.error("Error while stopping web app context for webapp " + webAppContext.getDisplayName(), e); - exception.addSuppressed(e); + exception = addMultiException(exception, e); } try { webServer.stop(); } catch (Exception e) { LOG.error("Error while stopping web server for webapp " + webAppContext.getDisplayName(), e); - exception.addSuppressed(e); + exception = addMultiException(exception, e); } if (exception != null) { - ifExceptionThrow(exception); + exception.ifExceptionThrow(); } } - /** - * Throw a {@link Throwable} as a checked {@link Exception} if it cannot be thrown as unchecked. - *

- * NOTE: This method is copied from Jetty-9 - * @param throwable The {@link Throwable} to throw or null. - * @throws Error If the passed {@link Throwable} is an {@link Error}. - * @throws Exception Otherwise, if the passed {@link Throwable} is not null. - */ - public static void ifExceptionThrow(Throwable throwable) throws Error, Exception { - if (throwable == null) { - return; - } - if (throwable instanceof Error error) { - throw error; - } - if (throwable instanceof Exception exception) { - throw exception; + private ExceptionUtil.MultiException addMultiException(ExceptionUtil.MultiException exception, Exception e) { + if (exception == null) { + exception = new ExceptionUtil.MultiException(); } - throw new RuntimeException(throwable); + exception.add(e); + return exception; } public void join() throws InterruptedException { From 4f6e62b2b57139fa0aac38b80c1819c10873f11e Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Sat, 29 Mar 2025 00:11:59 +0530 Subject: [PATCH 07/17] ee9 -> ee9+ --- .../src/test/resources/ensure-jars-have-correct-contents.sh | 2 +- .../src/test/resources/ensure-jars-have-correct-contents.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index c237ba9af7ed..0f854f5c2127 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -100,7 +100,7 @@ allowed_expr+="|^jetty-dir.css$" allowed_expr+="|^org/jspecify/$|^org/jspecify/annotations/$|^org/jspecify/annotations/.*\.class$" # Required by jetty 12 on ee8 allowed_expr="(|^javax/$)" -# Required by jetty 12 on ee9 +# Required by jetty 12 on ee9+ allowed_expr="(|^jakarta/$)" if [ -n "${allow_hadoop}" ]; then diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index c237ba9af7ed..0f854f5c2127 100644 --- a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -100,7 +100,7 @@ allowed_expr+="|^jetty-dir.css$" allowed_expr+="|^org/jspecify/$|^org/jspecify/annotations/$|^org/jspecify/annotations/.*\.class$" # Required by jetty 12 on ee8 allowed_expr="(|^javax/$)" -# Required by jetty 12 on ee9 +# Required by jetty 12 on ee9+ allowed_expr="(|^jakarta/$)" if [ -n "${allow_hadoop}" ]; then From ef20ec0873e8166f88af5186676b885f4dbb2217 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Fri, 4 Apr 2025 15:16:47 +0530 Subject: [PATCH 08/17] Fix TODOs, update log dir handling based on jetty project's feedback, add details on why we need violations --- .../apache/hadoop/hbase/http/HttpServer.java | 11 ++++++----- .../apache/hadoop/hbase/http/log/LogLevel.java | 4 ++-- .../apache/hadoop/hbase/rest/RESTServer.java | 18 ++++++++++++------ .../hbase/rest/TestGetAndPutResource.java | 3 +++ .../hbase/rest/client/TestRemoteTable.java | 17 ++++++++++++++++- .../ensure-jars-have-correct-contents.sh | 2 -- .../ensure-jars-have-correct-contents.sh | 2 -- 7 files changed, 39 insertions(+), 18 deletions(-) diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java index 0fa09ce6c570..efeab3ebe030 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java @@ -80,7 +80,6 @@ import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.webapp.WebAppContext; import org.apache.hbase.thirdparty.org.eclipse.jetty.http.HttpVersion; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Handler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ExceptionUtil; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConfiguration; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConnectionFactory; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.RequestLog; @@ -92,6 +91,7 @@ import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.ErrorHandler; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.gzip.GzipHandler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ExceptionUtil; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ssl.SslContextFactory; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool; import org.apache.hbase.thirdparty.org.glassfish.jersey.server.ResourceConfig; @@ -801,8 +801,8 @@ protected void addDefaultApps(ContextHandlerCollection parent, final String appD ServletContextHandler logContext = new ServletContextHandler(parent, "/logs"); logContext.addServlet(AdminAuthorizedServlet.class, "/*"); // We are doing this as otherwise jetty 12 is not handling /dir0/dir1/../dir2 - String logDirCanonical = Paths.get(logDir).toFile().getCanonicalPath(); - logContext.setResourceBase(logDirCanonical); + // See https://github.com/jetty/jetty.project/issues/12958 + logContext.setBaseResourceAsPath(Path.of(logDir).toRealPath()); logContext.setDisplayName("logs"); configureAliasChecks(logContext, conf.getBoolean(ServerConfigurationKeys.HBASE_JETTY_LOGS_SERVE_ALIASES, @@ -1259,7 +1259,7 @@ public void start() throws IOException { LOG.info("HttpServer.start() threw a non Bind IOException", ex); throw ex; } catch (Exception ex) { - LOG.info("HttpServer.start() threw a MultiException", ex); + LOG.info("HttpServer.start() threw a Exception", ex); throw ex; } // Make sure there is no handler failures. @@ -1372,7 +1372,8 @@ public void stop() throws Exception { } - private ExceptionUtil.MultiException addMultiException(ExceptionUtil.MultiException exception, Exception e) { + private ExceptionUtil.MultiException addMultiException(ExceptionUtil.MultiException exception, + Exception e) { if (exception == null) { exception = new ExceptionUtil.MultiException(); } diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java index 5db4ee1cac0a..915f7e299183 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java @@ -266,8 +266,8 @@ private void process(String urlString) throws Exception { HttpURLConnection connection = connect(url); - // We implement the validateResponse method inside hbase to handle for HTML response. - // as with Jetty 12: getResponseMessage returns "Precondition Failed" vs + // We now use the validateResponse method of hbase to handle for HTML response, + // as with Jetty 12: getResponseMessage() returns "Precondition Failed" vs // "Modification of logger protected.org.apache.hadoop.hbase.http.log.TestLogLevel is // disallowed in configuration" in Jetty 9 LogLevelExceptionUtils.validateResponse(connection, 200); diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java index e672370b2171..54da8c2a52ad 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java @@ -108,6 +108,9 @@ public class RESTServer implements Constants { static final String REST_CSRF_BROWSER_USERAGENTS_REGEX_KEY = "hbase.rest-csrf.browser-useragents-regex"; + static final String HTTP_SET_URI_COMPLIANCE = "hbase.rest.http.set.uri.compliance"; + static final boolean HTTP_SET_URI_COMPLIANCE_DEFAULT = true; + // HACK, making this static for AuthFilter to get at our configuration. Necessary for unit tests. @edu.umd.cs.findbugs.annotations.SuppressWarnings( value = { "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD", "MS_CANNOT_BE_FINAL" }, @@ -289,7 +292,15 @@ public synchronized void run() throws Exception { httpConfig.setResponseHeaderSize(DEFAULT_HTTP_MAX_HEADER_SIZE); httpConfig.setSendServerVersion(false); httpConfig.setSendDateHeader(false); - setUriComplianceRules(httpConfig); + + // In Jetty 12, ambiguous path separators, suspicious path characters, and ambiguous empty + // segments are considered violations of the URI specification and hence are not allowed. + // Refer to https://github.com/jetty/jetty.project/issues/11890#issuecomment-2156449534 + // We must set a URI compliance to allow for this violation so that client + // requests are not automatically rejected. We have tests which rely on this behavior. + if (conf.getBoolean(HTTP_SET_URI_COMPLIANCE, HTTP_SET_URI_COMPLIANCE_DEFAULT)) { + setUriComplianceRules(httpConfig); + } ServerConnector serverConnector; boolean isSecure = false; @@ -405,11 +416,6 @@ public synchronized void run() throws Exception { } private static void setUriComplianceRules(HttpConfiguration httpConfig) { - // In Jetty 12, ambiguous path separators, suspicious path characters, and ambiguous empty - // segments are considered violations of the URI specification and hence are not allowed. - // Refer to https://github.com/jetty/jetty.project/issues/11890#issuecomment-2156449534 - // We must set a URI compliance to allow for this violation so that client - // requests are not automatically rejected. We have tests which rely on this behavior. // TODO Discuss Should we set below to UriCompliance.LEGACY instead of cherry-picking? Set complianceViolationSet = new HashSet<>(); complianceViolationSet.add(UriCompliance.Violation.AMBIGUOUS_PATH_SEPARATOR); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java index 2751354f7418..21e948e3cfc8 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java @@ -323,6 +323,9 @@ public void testLatestCellGetJSON() throws IOException { public void testURLEncodedKey() throws IOException, JAXBException { // Requires UriCompliance.Violation.AMBIGUOUS_PATH_SEPARATOR // Otherwise fails with "400: Ambiguous URI path separator" + // In this test, request url resolves to "/TestRowResource/http%3A%2F%2Fexample.com%2Ffoo/a:1" + // and is considered ambiguous by Jetty 12. + // Basically we are having a URL encoded string as row key here! String urlKey = "http://example.com/foo"; StringBuilder path = new StringBuilder(); path.append('/'); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java index c33a42e1473d..4ed269746c83 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java @@ -166,6 +166,10 @@ public void testGetTableDescriptor() throws IOException { public void testGet() throws IOException { // Requires UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS // Otherwise fails with "400: Suspicious Path Character" + // In this test, the request path resolves to + // "/TestRemoteTable_-./testrow1%7C%22%5C%5E%7B%7D%01%02%03%04%05%06%07%08%09%0B%0C/" + // and is considered suspicious by the Jetty 12. + // Basically ROW_1 contains invalid URL characters here. Get get = new Get(ROW_1); Result result = remoteTable.get(get); byte[] value1 = result.getValue(COLUMN_1, QUALIFIER_1); @@ -266,6 +270,9 @@ public void testGet() throws IOException { @Test public void testMultiGet() throws Exception { + // In case of multi gets, the request path resolves to + // "/TestRemoteTable_-./multiget/?row=testrow1%7C%22%5C%5E&row=testrow2%7C%22%5C%5E%&v=3" + // and hence is not considered suspicious by the Jetty 12. ArrayList gets = new ArrayList<>(2); gets.add(new Get(ROW_1)); gets.add(new Get(ROW_2)); @@ -352,7 +359,11 @@ public void testPut() throws IOException { @Test public void testDelete() throws IOException { - // Requires UriCompliance.Violation.AMBIGUOUS_EMPTY_SEGMENT + // Requires UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS for put, + // otherwise fails with "400: Suspicious Path Character" + // This example is considered suspicious by the Jetty 12 due to reasons same as shown in testGet() + + // Also, requires UriCompliance.Violation.AMBIGUOUS_EMPTY_SEGMENT // Otherwise fails with "400: Ambiguous URI empty segment" Put put = new Put(ROW_3); put.addColumn(COLUMN_1, QUALIFIER_1, VALUE_1); @@ -393,6 +404,9 @@ public void testDelete() throws IOException { assertTrue(Bytes.equals(VALUE_1, value1)); assertNull(value2); + // This leads to path which resolves to + // "/TestRemoteTable_-./testrow3%7C%22%5C%5E%7B%7D%01%02%03%04%05%06%07%08%09%0B%0C//1" + // causing "400: Ambiguous URI empty segment" error with Jetty 12. delete = new Delete(ROW_3); delete.setTimestamp(1L); remoteTable.delete(delete); @@ -501,6 +515,7 @@ public void testScanner() throws IOException { public void testCheckAndDelete() throws IOException { // Requires UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS // Otherwise fails with "400: Suspicious Path Character" + // This example is considered suspicious by the Jetty 12 due to reasons same as shown in testGet() Get get = new Get(ROW_1); Result result = remoteTable.get(get); byte[] value1 = result.getValue(COLUMN_1, QUALIFIER_1); diff --git a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index 0f854f5c2127..8b63889f1950 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -100,8 +100,6 @@ allowed_expr+="|^jetty-dir.css$" allowed_expr+="|^org/jspecify/$|^org/jspecify/annotations/$|^org/jspecify/annotations/.*\.class$" # Required by jetty 12 on ee8 allowed_expr="(|^javax/$)" -# Required by jetty 12 on ee9+ -allowed_expr="(|^jakarta/$)" if [ -n "${allow_hadoop}" ]; then # * classes in packages that start with org.apache.hadoop, which by diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index 0f854f5c2127..8b63889f1950 100644 --- a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -100,8 +100,6 @@ allowed_expr+="|^jetty-dir.css$" allowed_expr+="|^org/jspecify/$|^org/jspecify/annotations/$|^org/jspecify/annotations/.*\.class$" # Required by jetty 12 on ee8 allowed_expr="(|^javax/$)" -# Required by jetty 12 on ee9+ -allowed_expr="(|^jakarta/$)" if [ -n "${allow_hadoop}" ]; then # * classes in packages that start with org.apache.hadoop, which by From abb677abec349b4a00963aa83ccd98387fca98e7 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Fri, 4 Apr 2025 15:20:24 +0530 Subject: [PATCH 09/17] Update comments, run spotless --- .../main/java/org/apache/hadoop/hbase/rest/RESTServer.java | 5 +++-- .../apache/hadoop/hbase/rest/client/TestRemoteTable.java | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java index 54da8c2a52ad..a38c8c8f80bd 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java @@ -296,8 +296,9 @@ public synchronized void run() throws Exception { // In Jetty 12, ambiguous path separators, suspicious path characters, and ambiguous empty // segments are considered violations of the URI specification and hence are not allowed. // Refer to https://github.com/jetty/jetty.project/issues/11890#issuecomment-2156449534 - // We must set a URI compliance to allow for this violation so that client - // requests are not automatically rejected. We have tests which rely on this behavior. + // We must set a URI compliance to allow for this violation so that client requests are not + // automatically rejected. Our rest endpoints rely on this behavior to handle encoded uri paths. + // Optionally, we can decide to not set this compliance rules, but may break existing clients. if (conf.getBoolean(HTTP_SET_URI_COMPLIANCE, HTTP_SET_URI_COMPLIANCE_DEFAULT)) { setUriComplianceRules(httpConfig); } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java index 4ed269746c83..c464a51813e7 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java @@ -361,7 +361,8 @@ public void testPut() throws IOException { public void testDelete() throws IOException { // Requires UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS for put, // otherwise fails with "400: Suspicious Path Character" - // This example is considered suspicious by the Jetty 12 due to reasons same as shown in testGet() + // This example is considered suspicious by the Jetty 12 due to reasons same as shown in + // testGet() // Also, requires UriCompliance.Violation.AMBIGUOUS_EMPTY_SEGMENT // Otherwise fails with "400: Ambiguous URI empty segment" @@ -515,7 +516,8 @@ public void testScanner() throws IOException { public void testCheckAndDelete() throws IOException { // Requires UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS // Otherwise fails with "400: Suspicious Path Character" - // This example is considered suspicious by the Jetty 12 due to reasons same as shown in testGet() + // This example is considered suspicious by the Jetty 12 due to reasons same as shown in + // testGet() Get get = new Get(ROW_1); Result result = remoteTable.get(get); byte[] value1 = result.getValue(COLUMN_1, QUALIFIER_1); From a7519adb42cb0c76f4aacde8c12fcbf440d78b4c Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Sat, 5 Apr 2025 00:46:50 +0530 Subject: [PATCH 10/17] Add snapshot repo --- pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pom.xml b/pom.xml index 056abf7b836e..1f34da86c5a3 100644 --- a/pom.xml +++ b/pom.xml @@ -4987,4 +4987,14 @@ + + + + apache.snapshots + https://repository.apache.org/content/repositories/snapshots + + true + + + From 1f54e07f6564b955bc054628ba77686af682c0d1 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Sat, 5 Apr 2025 13:27:51 +0530 Subject: [PATCH 11/17] Set update policy to always to force update --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 1f34da86c5a3..1757e2db955d 100644 --- a/pom.xml +++ b/pom.xml @@ -4994,6 +4994,7 @@ https://repository.apache.org/content/repositories/snapshots true + always From 25ec1950e822b639f59cc9475b4b1983b0b57105 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Sat, 5 Apr 2025 16:09:19 +0530 Subject: [PATCH 12/17] Fix spotbugs and spotless issues --- .../http/log/LogLevelExceptionUtils.java | 21 +++++++++--------- pom.xml | 22 +++++++++---------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java index f19085943705..2776c9b4313b 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java @@ -63,18 +63,19 @@ public static void validateResponse(HttpURLConnection conn, int expectedStatus) try (InputStream es = conn.getErrorStream()) { if (es != null) { - final String errorAsHtml = - new BufferedReader(new InputStreamReader(es, StandardCharsets.UTF_8)).lines() - .collect(Collectors.joining("\n")); + try (InputStreamReader isr = new InputStreamReader(es, StandardCharsets.UTF_8); + BufferedReader reader = new BufferedReader(isr)) { + final String errorAsHtml = reader.lines().collect(Collectors.joining("\n")); - final String status = extractValue(errorAsHtml, "STATUS:(\\d+)"); - final String message = extractValue(errorAsHtml, "MESSAGE:([^<]+)"); - final String uri = extractValue(errorAsHtml, "URI:([^<]+)"); - final String exception = extractValue(errorAsHtml, "([^<]+)"); + final String status = extractValue(errorAsHtml, "STATUS:(\\d+)"); + final String message = extractValue(errorAsHtml, "MESSAGE:([^<]+)"); + final String uri = extractValue(errorAsHtml, "URI:([^<]+)"); + final String exception = extractValue(errorAsHtml, "([^<]+)"); - toThrow = new IOException( - String.format("HTTP status [%s], message [%s], URL [%s], exception [%s]", status, - message, uri, exception)); + toThrow = new IOException( + String.format("HTTP status [%s], message [%s], URL [%s], exception [%s]", status, + message, uri, exception)); + } } } catch (Exception ex) { toThrow = diff --git a/pom.xml b/pom.xml index 1757e2db955d..7c7a66f48f73 100644 --- a/pom.xml +++ b/pom.xml @@ -1909,6 +1909,17 @@ test + + + + + true + always + + apache.snapshots + https://repository.apache.org/content/repositories/snapshots + + - - - apache.snapshots - https://repository.apache.org/content/repositories/snapshots - - true - always - - - From 527d27e88427edb4e07e10ea4c120dc703766baf Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Mon, 7 Apr 2025 11:49:59 +0530 Subject: [PATCH 13/17] Remove check to optionally enable full compliance, and remove todo for LEGACY disuss --- .../java/org/apache/hadoop/hbase/rest/RESTServer.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java index a38c8c8f80bd..666c994d6924 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java @@ -108,9 +108,6 @@ public class RESTServer implements Constants { static final String REST_CSRF_BROWSER_USERAGENTS_REGEX_KEY = "hbase.rest-csrf.browser-useragents-regex"; - static final String HTTP_SET_URI_COMPLIANCE = "hbase.rest.http.set.uri.compliance"; - static final boolean HTTP_SET_URI_COMPLIANCE_DEFAULT = true; - // HACK, making this static for AuthFilter to get at our configuration. Necessary for unit tests. @edu.umd.cs.findbugs.annotations.SuppressWarnings( value = { "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD", "MS_CANNOT_BE_FINAL" }, @@ -298,10 +295,7 @@ public synchronized void run() throws Exception { // Refer to https://github.com/jetty/jetty.project/issues/11890#issuecomment-2156449534 // We must set a URI compliance to allow for this violation so that client requests are not // automatically rejected. Our rest endpoints rely on this behavior to handle encoded uri paths. - // Optionally, we can decide to not set this compliance rules, but may break existing clients. - if (conf.getBoolean(HTTP_SET_URI_COMPLIANCE, HTTP_SET_URI_COMPLIANCE_DEFAULT)) { - setUriComplianceRules(httpConfig); - } + setUriComplianceRules(httpConfig); ServerConnector serverConnector; boolean isSecure = false; @@ -417,7 +411,6 @@ public synchronized void run() throws Exception { } private static void setUriComplianceRules(HttpConfiguration httpConfig) { - // TODO Discuss Should we set below to UriCompliance.LEGACY instead of cherry-picking? Set complianceViolationSet = new HashSet<>(); complianceViolationSet.add(UriCompliance.Violation.AMBIGUOUS_PATH_SEPARATOR); complianceViolationSet.add(UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS); From dbff9fa105b541bba41b7d4e70b1b41d4c6930db Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Wed, 7 May 2025 16:42:36 +0530 Subject: [PATCH 14/17] Jump to jetty 12.0.20 which brings in fix for https://github.com/jetty/jetty.project/issues/12958 - Discrepancy between Jetty 9 and Jetty 12 when setting the base resource to a path containing .. --- .../main/java/org/apache/hadoop/hbase/http/HttpServer.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java index efeab3ebe030..fe2a9a48c210 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java @@ -791,7 +791,7 @@ private static FilterInitializer[] getFilterInitializers(Configuration conf) { * @param appDir The application directory */ protected void addDefaultApps(ContextHandlerCollection parent, final String appDir, - Configuration conf) throws IOException { + Configuration conf) { // set up the context for "/logs/" if "hadoop.log.dir" property is defined. String logDir = this.logDir; if (logDir == null) { @@ -800,9 +800,7 @@ protected void addDefaultApps(ContextHandlerCollection parent, final String appD if (logDir != null) { ServletContextHandler logContext = new ServletContextHandler(parent, "/logs"); logContext.addServlet(AdminAuthorizedServlet.class, "/*"); - // We are doing this as otherwise jetty 12 is not handling /dir0/dir1/../dir2 - // See https://github.com/jetty/jetty.project/issues/12958 - logContext.setBaseResourceAsPath(Path.of(logDir).toRealPath()); + logContext.setResourceBase(logDir); logContext.setDisplayName("logs"); configureAliasChecks(logContext, conf.getBoolean(ServerConfigurationKeys.HBASE_JETTY_LOGS_SERVE_ALIASES, From 8c496dcbbe2cd7dc2e7f05df5793fe17b99c27d9 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Thu, 31 Jul 2025 01:47:21 +0530 Subject: [PATCH 15/17] Use with hbase-thirdparty 4.1.12-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7c7a66f48f73..b362e5d81985 100644 --- a/pom.xml +++ b/pom.xml @@ -895,7 +895,7 @@ 2.3.1 4.0.1 2.1.1 - 9.0.104 + 9.0.107 9.4.12.1 4.13.2 1.3 From d4a934622a80d2c282fe2843c8c2299298914afc Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Fri, 8 Aug 2025 21:10:56 +0530 Subject: [PATCH 16/17] Consume hbase-thirdparty 4.1.12RC0 --- pom.xml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index b362e5d81985..5186930b73cf 100644 --- a/pom.xml +++ b/pom.xml @@ -895,7 +895,7 @@ 2.3.1 4.0.1 2.1.1 - 9.0.107 + 9.0.108 9.4.12.1 4.13.2 1.3 @@ -1912,12 +1912,9 @@ - - true - always - - apache.snapshots - https://repository.apache.org/content/repositories/snapshots + staging + staging + https://repository.apache.org/content/repositories/orgapachehbase-1586 From 1c8dfa1db9b3e956dc219d6a7e74c503fee68f1f Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Sun, 17 Aug 2025 18:42:15 +0530 Subject: [PATCH 17/17] Drop staging repo --- pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pom.xml b/pom.xml index 5186930b73cf..bb24fe435cf3 100644 --- a/pom.xml +++ b/pom.xml @@ -1909,14 +1909,6 @@ test - - - - staging - staging - https://repository.apache.org/content/repositories/orgapachehbase-1586 - -