From e890636668acadfee93003a66f64f3ae49d590ea Mon Sep 17 00:00:00 2001 From: arushir Date: Tue, 10 Sep 2019 11:39:55 -0400 Subject: [PATCH 1/6] Added memory leak fix in the ClientToProxyConnection.java based on issue reported in https://github.com/mrog/LittleProxy/issues/32 --- README.md | 4 ++++ pom.xml | 2 +- .../org/littleshoot/proxy/impl/ClientToProxyConnection.java | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d512855e6..2b246a1bf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ [![Build Status](https://travis-ci.org/adamfisk/LittleProxy.png?branch=master)](https://travis-ci.org/adamfisk/LittleProxy) +**This branch is used by MAG proxy which the following custom changes: +- NTLM support +- Memory Leak Fix** + LittleProxy is a high performance HTTP proxy written in Java atop Trustin Lee's excellent [Netty](http://netty.io) event-based networking library. It's quite stable, performs well, and is easy to integrate into your projects. One option is to clone LittleProxy and run it from the command line. This is as simple as: diff --git a/pom.xml b/pom.xml index 6e933633d..e5b8cd8af 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.littleshoot littleproxy jar - 1.1.2-ntlm-20180920 + 1.1.2-20190910 LittleProxy LittleProxy is a high performance HTTP proxy written in Java and using the Netty networking framework. diff --git a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java index 17e4ba05b..19012d378 100644 --- a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java +++ b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java @@ -21,6 +21,7 @@ import io.netty.handler.codec.http.HttpVersion; import io.netty.handler.timeout.IdleStateHandler; import io.netty.handler.traffic.GlobalTrafficShapingHandler; +import io.netty.util.ReferenceCounted; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.GenericFutureListener; import org.apache.commons.lang3.StringUtils; @@ -434,6 +435,9 @@ void respond(ProxyToServerConnection serverConnection, HttpFilters filters, HttpRequest currentHttpRequest, HttpResponse currentHttpResponse, HttpObject httpObject) { // we are sending a response to the client, so we are done handling this request + if (currentRequest != null && currentRequest instanceof ReferenceCounted) { + ((ReferenceCounted)currentRequest).release(); + } this.currentRequest = null; httpObject = filters.serverToProxyResponse(httpObject); From f258107a9cfa7b44f4f380a63114298ae2e8b5b2 Mon Sep 17 00:00:00 2001 From: arushir Date: Tue, 10 Sep 2019 16:37:16 -0400 Subject: [PATCH 2/6] Updated littleproxy version Release currentRequest in respondWithShortCircuitResponse method --- pom.xml | 2 +- .../org/littleshoot/proxy/impl/ClientToProxyConnection.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e5b8cd8af..c0c9902ee 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.littleshoot littleproxy jar - 1.1.2-20190910 + 1.1.2-ntlm-20190910 LittleProxy LittleProxy is a high performance HTTP proxy written in Java and using the Netty networking framework. diff --git a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java index 19012d378..57865e673 100644 --- a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java +++ b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java @@ -1276,6 +1276,9 @@ private boolean writeGatewayTimeout(HttpRequest httpRequest) { */ private boolean respondWithShortCircuitResponse(HttpResponse httpResponse) { // we are sending a response to the client, so we are done handling this request + if (currentRequest != null && currentRequest instanceof ReferenceCounted) { + ((ReferenceCounted)currentRequest).release(); + } this.currentRequest = null; HttpResponse filteredResponse = (HttpResponse) currentFilters.proxyToClientResponse(httpResponse); From de594ba5f84a9c444b896f901a726854d6a03fc5 Mon Sep 17 00:00:00 2001 From: arushir Date: Fri, 4 Oct 2019 11:46:39 -0400 Subject: [PATCH 3/6] Updated netty version Took ConnectionFlow fix from https://github.com/verygoodsecurity/LittleProxy/pull/48/commits --- pom.xml | 4 +- .../proxy/impl/ConnectionFlow.java | 52 +++++++++++++------ .../proxy/impl/ProxyToServerConnection.java | 7 ++- 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index c0c9902ee..37ee89ad4 100644 --- a/pom.xml +++ b/pom.xml @@ -14,9 +14,9 @@ UTF-8 UTF-8 github - 4.0.44.Final + 4.1.28.Final 1.7.24 - 1.7 + 1.8 diff --git a/src/main/java/org/littleshoot/proxy/impl/ConnectionFlow.java b/src/main/java/org/littleshoot/proxy/impl/ConnectionFlow.java index 26c7db952..c8d7c8735 100644 --- a/src/main/java/org/littleshoot/proxy/impl/ConnectionFlow.java +++ b/src/main/java/org/littleshoot/proxy/impl/ConnectionFlow.java @@ -1,5 +1,6 @@ package org.littleshoot.proxy.impl; +import io.netty.util.ReferenceCounted; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.GenericFutureListener; @@ -177,11 +178,20 @@ public void operationComplete( */ void succeed() { synchronized (connectLock) { - serverConnection.getLOG().debug( - "Connection flow completed successfully: {}", currentStep); - serverConnection.connectionSucceeded(!suppressInitialRequest); - notifyThreadsWaitingForConnection(); - } + try { + serverConnection.getLOG().debug( + "Connection flow completed successfully: {}", currentStep); + serverConnection.connectionSucceeded(!suppressInitialRequest); + notifyThreadsWaitingForConnection(); + } finally { + // we're now done with the initialRequest: it's either been forwarded to the upstream server (HTTP requests), or + // completely dropped (HTTPS CONNECTs). if the initialRequest is reference counted (typically because the HttpObjectAggregator is in + // the pipeline to generate FullHttpRequests), we need to manually release it to avoid a memory leak. + if (serverConnection.getInitialRequest() instanceof ReferenceCounted) { + ((ReferenceCounted)serverConnection.getInitialRequest()).release(); + } + } + } } /** @@ -199,16 +209,28 @@ void fail(final Throwable cause) { public void operationComplete(Future future) throws Exception { synchronized (connectLock) { - if (!clientConnection.serverConnectionFailed( - serverConnection, - lastStateBeforeFailure, - cause)) { - // the connection to the server failed and we are not retrying, so transition to the - // DISCONNECTED state - serverConnection.become(ConnectionState.DISCONNECTED); - - // We are not retrying our connection, let anyone waiting for a connection know that we're done - notifyThreadsWaitingForConnection(); + + boolean fallbackToAnotherChainedProxy = false; + + try { + fallbackToAnotherChainedProxy = clientConnection.serverConnectionFailed( + serverConnection, + lastStateBeforeFailure, + cause); + } finally { + // Do not release when there is fallback chained proxy + if (!fallbackToAnotherChainedProxy) { + if (serverConnection.getInitialRequest() instanceof ReferenceCounted) { + ((ReferenceCounted)serverConnection.getInitialRequest()).release(); + } + + // the connection to the server failed and we are not retrying, so transition to the + // DISCONNECTED state + serverConnection.become(ConnectionState.DISCONNECTED); + + // We are not retrying our connection, let anyone waiting for a connection know that we're done + notifyThreadsWaitingForConnection(); + } } } } diff --git a/src/main/java/org/littleshoot/proxy/impl/ProxyToServerConnection.java b/src/main/java/org/littleshoot/proxy/impl/ProxyToServerConnection.java index 79a7e52ac..189ed09c2 100644 --- a/src/main/java/org/littleshoot/proxy/impl/ProxyToServerConnection.java +++ b/src/main/java/org/littleshoot/proxy/impl/ProxyToServerConnection.java @@ -383,6 +383,10 @@ void write(Object msg) { // already disconnected if (isConnecting() || getCurrentState().isDisconnectingOrDisconnected()) { LOG.debug("Connection failed or timed out while waiting to write message to server. Message will be discarded: {}", msg); + // release when disconnected. + if (initialRequest instanceof ReferenceCounted) { + ((ReferenceCounted)initialRequest).release(); + } return; } @@ -397,9 +401,8 @@ protected void writeHttp(HttpObject httpObject) { chainedProxy.filterRequest(httpObject); } if (httpObject instanceof HttpRequest) { - HttpRequest httpRequest = (HttpRequest) httpObject; // Remember that we issued this HttpRequest for later - currentHttpRequest = httpRequest; + currentHttpRequest = (HttpRequest) httpObject; } super.writeHttp(httpObject); } From 4c35cda799225c4dce27caa565d5f2b6779cd544 Mon Sep 17 00:00:00 2001 From: arushir Date: Mon, 14 Oct 2019 13:19:24 -0400 Subject: [PATCH 4/6] Updated netty version to 4.1.42 Commented activity tracker code --- pom.xml | 2 +- .../proxy/impl/ClientToProxyConnection.java | 12 +++---- .../proxy/impl/ProxyConnection.java | 36 +++++++++---------- .../proxy/impl/ProxyToServerConnection.java | 16 +++++---- 4 files changed, 34 insertions(+), 32 deletions(-) diff --git a/pom.xml b/pom.xml index 37ee89ad4..9e227df24 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ UTF-8 UTF-8 github - 4.1.28.Final + 4.1.42.Final 1.7.24 1.8 diff --git a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java index 57865e673..8e1850ca8 100644 --- a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java +++ b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java @@ -788,8 +788,8 @@ protected void exceptionCaught(Throwable cause) { private void initChannelPipeline(ChannelPipeline pipeline) { LOG.debug("Configuring ChannelPipeline"); - pipeline.addLast("bytesReadMonitor", bytesReadMonitor); - pipeline.addLast("bytesWrittenMonitor", bytesWrittenMonitor); +// pipeline.addLast("bytesReadMonitor", bytesReadMonitor); +// pipeline.addLast("bytesWrittenMonitor", bytesWrittenMonitor); pipeline.addLast("encoder", new HttpResponseEncoder()); // We want to allow longer request lines, headers, and chunks @@ -806,8 +806,8 @@ private void initChannelPipeline(ChannelPipeline pipeline) { aggregateContentForFiltering(pipeline, numberOfBytesToBuffer); } - pipeline.addLast("requestReadMonitor", requestReadMonitor); - pipeline.addLast("responseWrittenMonitor", responseWrittenMonitor); +// pipeline.addLast("requestReadMonitor", requestReadMonitor); +// pipeline.addLast("responseWrittenMonitor", responseWrittenMonitor); pipeline.addLast( "idle", @@ -1367,7 +1367,7 @@ protected void setMitming(boolean isMitming) { * We track statistics on bytes, requests and responses by adding handlers * at the appropriate parts of the pipeline (see initChannelPipeline()). **************************************************************************/ - private final BytesReadMonitor bytesReadMonitor = new BytesReadMonitor() { + /*private final BytesReadMonitor bytesReadMonitor = new BytesReadMonitor() { @Override protected void bytesRead(int numberOfBytes) { FlowContext flowContext = flowContext(); @@ -1410,7 +1410,7 @@ protected void responseWritten(HttpResponse httpResponse) { httpResponse); } } - }; + };*/ private void recordClientConnected() { try { diff --git a/src/main/java/org/littleshoot/proxy/impl/ProxyConnection.java b/src/main/java/org/littleshoot/proxy/impl/ProxyConnection.java index f8490d5e5..21a97af4f 100644 --- a/src/main/java/org/littleshoot/proxy/impl/ProxyConnection.java +++ b/src/main/java/org/littleshoot/proxy/impl/ProxyConnection.java @@ -676,7 +676,7 @@ public final void userEventTriggered(ChannelHandlerContext ctx, Object evt) /** * Utility handler for monitoring bytes read on this connection. */ - @Sharable + /*@Sharable protected abstract class BytesReadMonitor extends ChannelInboundHandlerAdapter { @Override @@ -696,9 +696,9 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) protected abstract void bytesRead(int numberOfBytes); } - /** + *//** * Utility handler for monitoring requests read on this connection. - */ + *//* @Sharable protected abstract class RequestReadMonitor extends ChannelInboundHandlerAdapter { @@ -719,9 +719,9 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) protected abstract void requestRead(HttpRequest httpRequest); } - /** + *//** * Utility handler for monitoring responses read on this connection. - */ + *//* @Sharable protected abstract class ResponseReadMonitor extends ChannelInboundHandlerAdapter { @@ -742,9 +742,9 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) protected abstract void responseRead(HttpResponse httpResponse); } - /** + *//** * Utility handler for monitoring bytes written on this connection. - */ + *//* @Sharable protected abstract class BytesWrittenMonitor extends ChannelOutboundHandlerAdapter { @@ -766,9 +766,9 @@ public void write(ChannelHandlerContext ctx, protected abstract void bytesWritten(int numberOfBytes); } - /** + *//** * Utility handler for monitoring requests written on this connection. - */ + *//* @Sharable protected abstract class RequestWrittenMonitor extends ChannelOutboundHandlerAdapter { @@ -796,25 +796,25 @@ public void write(ChannelHandlerContext ctx, } } - /** + *//** * Invoked immediately before an HttpRequest is written. - */ + *//* protected abstract void requestWriting(HttpRequest httpRequest); - /** + *//** * Invoked immediately after an HttpRequest has been sent. - */ + *//* protected abstract void requestWritten(HttpRequest httpRequest); - /** + *//** * Invoked immediately after an HttpContent has been sent. - */ + *//* protected abstract void contentWritten(HttpContent httpContent); } - /** + *//** * Utility handler for monitoring responses written on this connection. - */ + *//* @Sharable protected abstract class ResponseWrittenMonitor extends ChannelOutboundHandlerAdapter { @@ -835,7 +835,7 @@ public void write(ChannelHandlerContext ctx, protected abstract void responseWritten(HttpResponse httpResponse); } - +*/ public ChannelHandlerContext getContext() { return ctx; } diff --git a/src/main/java/org/littleshoot/proxy/impl/ProxyToServerConnection.java b/src/main/java/org/littleshoot/proxy/impl/ProxyToServerConnection.java index 189ed09c2..cce656896 100644 --- a/src/main/java/org/littleshoot/proxy/impl/ProxyToServerConnection.java +++ b/src/main/java/org/littleshoot/proxy/impl/ProxyToServerConnection.java @@ -354,7 +354,9 @@ void write(Object msg, HttpFilters filters) { @Override void write(Object msg) { - LOG.debug("Requested write of {}", msg); + //TODO: Need to revert them to DEBUG + LOG.info("Requested write of {}", msg); + LOG.info("Message Instance Type - {}", msg.getClass()); if (msg instanceof ReferenceCounted) { LOG.debug("Retaining reference counted message"); @@ -1008,8 +1010,8 @@ private void initChannelPipeline(ChannelPipeline pipeline, pipeline.addLast("global-traffic-shaping", trafficHandler); } - pipeline.addLast("bytesReadMonitor", bytesReadMonitor); - pipeline.addLast("bytesWrittenMonitor", bytesWrittenMonitor); +// pipeline.addLast("bytesReadMonitor", bytesReadMonitor); +// pipeline.addLast("bytesWrittenMonitor", bytesWrittenMonitor); pipeline.addLast("encoder", new HttpRequestEncoder()); pipeline.addLast("decoder", new HeadAwareHttpResponseDecoder( @@ -1024,8 +1026,8 @@ private void initChannelPipeline(ChannelPipeline pipeline, aggregateContentForFiltering(pipeline, numberOfBytesToBuffer); } - pipeline.addLast("responseReadMonitor", responseReadMonitor); - pipeline.addLast("requestWrittenMonitor", requestWrittenMonitor); +// pipeline.addLast("responseReadMonitor", responseReadMonitor); +// pipeline.addLast("requestWrittenMonitor", requestWrittenMonitor); // Set idle timeout pipeline.addLast( @@ -1126,7 +1128,7 @@ public static InetSocketAddress addressFor(String hostAndPort, DefaultHttpProxyS * We track statistics on bytes, requests and responses by adding handlers * at the appropriate parts of the pipeline (see initChannelPipeline()). **************************************************************************/ - private final BytesReadMonitor bytesReadMonitor = new BytesReadMonitor() { + /*private final BytesReadMonitor bytesReadMonitor = new BytesReadMonitor() { @Override protected void bytesRead(int numberOfBytes) { FullFlowContext flowContext = new FullFlowContext(clientConnection, @@ -1189,6 +1191,6 @@ protected void contentWritten(HttpContent httpContent) { currentFilters.proxyToServerRequestSent(); } } - }; + };*/ } From 544ba904136cb9ebf4b316c348fde05a7ed8c584 Mon Sep 17 00:00:00 2001 From: arushir Date: Fri, 25 Oct 2019 09:32:13 -0400 Subject: [PATCH 5/6] Use ReferenceCountUtil instead for releasing currentRequest --- .../littleshoot/proxy/impl/ClientToProxyConnection.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java index 8e1850ca8..50b65fa6c 100644 --- a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java +++ b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java @@ -21,6 +21,7 @@ import io.netty.handler.codec.http.HttpVersion; import io.netty.handler.timeout.IdleStateHandler; import io.netty.handler.traffic.GlobalTrafficShapingHandler; +import io.netty.util.ReferenceCountUtil; import io.netty.util.ReferenceCounted; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.GenericFutureListener; @@ -435,9 +436,7 @@ void respond(ProxyToServerConnection serverConnection, HttpFilters filters, HttpRequest currentHttpRequest, HttpResponse currentHttpResponse, HttpObject httpObject) { // we are sending a response to the client, so we are done handling this request - if (currentRequest != null && currentRequest instanceof ReferenceCounted) { - ((ReferenceCounted)currentRequest).release(); - } + ReferenceCountUtil.release(currentRequest); this.currentRequest = null; httpObject = filters.serverToProxyResponse(httpObject); @@ -1276,9 +1275,7 @@ private boolean writeGatewayTimeout(HttpRequest httpRequest) { */ private boolean respondWithShortCircuitResponse(HttpResponse httpResponse) { // we are sending a response to the client, so we are done handling this request - if (currentRequest != null && currentRequest instanceof ReferenceCounted) { - ((ReferenceCounted)currentRequest).release(); - } + ReferenceCountUtil.release(currentRequest); this.currentRequest = null; HttpResponse filteredResponse = (HttpResponse) currentFilters.proxyToClientResponse(httpResponse); From 112b7fc162dc9aa5f418f5e4b127d35a15aa06b2 Mon Sep 17 00:00:00 2001 From: arushir Date: Fri, 25 Oct 2019 13:49:48 -0400 Subject: [PATCH 6/6] added a check --- .../org/littleshoot/proxy/impl/ClientToProxyConnection.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java index 50b65fa6c..a699b5082 100644 --- a/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java +++ b/src/main/java/org/littleshoot/proxy/impl/ClientToProxyConnection.java @@ -874,7 +874,7 @@ private boolean shouldCloseClientConnection(HttpRequest req, } } - if (!HttpHeaders.isKeepAlive(req)) { + if (req != null && !HttpHeaders.isKeepAlive(req)) { LOG.debug("Closing client connection since request is not keep alive: {}", req); // Here we simply want to close the connection because the // client itself has requested it be closed in the request.