From 21e67b8c574dbeb68533da7fb7e558fa2e156d18 Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Wed, 5 Feb 2014 09:41:55 -0800 Subject: [PATCH 01/16] Reorder some functions so related functions are grouped together --- plugins/experimental/metalink/metalink.cc | 94 +++++++++++------------ 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/plugins/experimental/metalink/metalink.cc b/plugins/experimental/metalink/metalink.cc index 18ad584a053..84d5876b746 100644 --- a/plugins/experimental/metalink/metalink.cc +++ b/plugins/experimental/metalink/metalink.cc @@ -301,39 +301,23 @@ transform_handler(TSCont contp, TSEvent event, void *edata) return 0; } +/* Compute SHA-256 digest, write to cache, and store there the request URL */ + static int -rewrite_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) +http_read_response_hdr(TSCont /* contp ATS_UNUSED */, void *edata) { - const char *value; - int length; - - SendData *data = (SendData *) TSContDataGet(contp); - TSContDestroy(contp); - - switch (event) { - - /* Yes: Rewrite "Location: ..." header and reenable response */ - case TS_EVENT_CACHE_OPEN_READ: - value = TSUrlStringGet(data->resp_bufp, data->url_loc, &length); - TSMimeHdrFieldValuesClear(data->resp_bufp, data->hdr_loc, data->location_loc); - TSMimeHdrFieldValueStringInsert(data->resp_bufp, data->hdr_loc, data->location_loc, -1, value, length); - break; - - case TS_EVENT_CACHE_OPEN_READ_FAILED: - break; + TransformData *data = (TransformData *) TSmalloc(sizeof(TransformData)); + data->txnp = (TSHttpTxn) edata; - default: - TSAssert(!"Unexpected event"); - } + /* Can't TSVConnWrite() before TS_HTTP_RESPONSE_TRANSFORM_HOOK */ + data->bufp = NULL; - TSCacheKeyDestroy(data->key); + TSVConn connp = TSTransformCreate(transform_handler, data->txnp); + TSContDataSet(connp, data); - TSHandleMLocRelease(data->resp_bufp, TS_NULL_MLOC, data->url_loc); - TSHandleMLocRelease(data->resp_bufp, data->hdr_loc, data->location_loc); - TSHandleMLocRelease(data->resp_bufp, TS_NULL_MLOC, data->hdr_loc); + TSHttpTxnHookAdd(data->txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp); TSHttpTxnReenable(data->txnp, TS_EVENT_HTTP_CONTINUE); - TSfree(data); return 0; } @@ -368,6 +352,43 @@ cache_open_read_failed(TSCont contp, void * /* edata ATS_UNUSED */) return 0; } +static int +rewrite_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) +{ + const char *value; + int length; + + SendData *data = (SendData *) TSContDataGet(contp); + TSContDestroy(contp); + + switch (event) { + + /* Yes: Rewrite "Location: ..." header and reenable response */ + case TS_EVENT_CACHE_OPEN_READ: + value = TSUrlStringGet(data->resp_bufp, data->url_loc, &length); + TSMimeHdrFieldValuesClear(data->resp_bufp, data->hdr_loc, data->location_loc); + TSMimeHdrFieldValueStringInsert(data->resp_bufp, data->hdr_loc, data->location_loc, -1, value, length); + break; + + case TS_EVENT_CACHE_OPEN_READ_FAILED: + break; + + default: + TSAssert(!"Unexpected event"); + } + + TSCacheKeyDestroy(data->key); + + TSHandleMLocRelease(data->resp_bufp, TS_NULL_MLOC, data->url_loc); + TSHandleMLocRelease(data->resp_bufp, data->hdr_loc, data->location_loc); + TSHandleMLocRelease(data->resp_bufp, TS_NULL_MLOC, data->hdr_loc); + + TSHttpTxnReenable(data->txnp, TS_EVENT_HTTP_CONTINUE); + TSfree(data); + + return 0; +} + static int vconn_read_ready(TSCont contp, void * /* edata ATS_UNUSED */) { @@ -497,27 +518,6 @@ location_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) return 0; } -/* Compute SHA-256 digest, write to cache, and store there the request URL */ - -static int -http_read_response_hdr(TSCont /* contp ATS_UNUSED */, void *edata) -{ - TransformData *data = (TransformData *) TSmalloc(sizeof(TransformData)); - data->txnp = (TSHttpTxn) edata; - - /* Can't TSVConnWrite() before TS_HTTP_RESPONSE_TRANSFORM_HOOK */ - data->bufp = NULL; - - TSVConn connp = TSTransformCreate(transform_handler, data->txnp); - TSContDataSet(connp, data); - - TSHttpTxnHookAdd(data->txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp); - - TSHttpTxnReenable(data->txnp, TS_EVENT_HTTP_CONTINUE); - - return 0; -} - static int http_send_response_hdr(TSCont contp, void *edata) { From 8240d6584b016b962bb92a9108a82d38b27330ce Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Wed, 5 Feb 2014 09:53:02 -0800 Subject: [PATCH 02/16] Rename some variables to more easily disambiguate them --- plugins/experimental/metalink/metalink.cc | 74 +++++++++++------------ 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/plugins/experimental/metalink/metalink.cc b/plugins/experimental/metalink/metalink.cc index 84d5876b746..afb582c222f 100644 --- a/plugins/experimental/metalink/metalink.cc +++ b/plugins/experimental/metalink/metalink.cc @@ -48,8 +48,8 @@ typedef struct { TSHttpTxn txnp; /* Null transform */ - TSIOBuffer bufp; - TSVIO viop; + TSIOBuffer output_bufp; + TSVIO output_viop; /* Message digest handle */ SHA256_CTX c; @@ -115,7 +115,7 @@ write_handler(TSCont contp, TSEvent event, void *edata) static int cache_open_write(TSCont contp, void *edata) { - TSMBuffer bufp; + TSMBuffer req_bufp; TSMLoc hdr_loc; TSMLoc url_loc; @@ -136,28 +136,28 @@ cache_open_write(TSCont contp, void *edata) write_data->bufp = TSIOBufferCreate(); TSIOBufferReader readerp = TSIOBufferReaderAlloc(write_data->bufp); - if (TSHttpTxnClientReqGet(transform_data->txnp, &bufp, &hdr_loc) != TS_SUCCESS) { + if (TSHttpTxnClientReqGet(transform_data->txnp, &req_bufp, &hdr_loc) != TS_SUCCESS) { TSError("Couldn't retrieve client request header"); return 0; } - if (TSHttpHdrUrlGet(bufp, hdr_loc, &url_loc) != TS_SUCCESS) { - TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); + if (TSHttpHdrUrlGet(req_bufp, hdr_loc, &url_loc) != TS_SUCCESS) { + TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, hdr_loc); return 0; } - value = TSUrlStringGet(bufp, url_loc, &length); + value = TSUrlStringGet(req_bufp, url_loc, &length); if (!value) { - TSHandleMLocRelease(bufp, hdr_loc, url_loc); - TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); + TSHandleMLocRelease(req_bufp, hdr_loc, url_loc); + TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, hdr_loc); return 0; } - TSHandleMLocRelease(bufp, hdr_loc, url_loc); - TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); + TSHandleMLocRelease(req_bufp, hdr_loc, url_loc); + TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, hdr_loc); int nbytes = TSIOBufferWrite(write_data->bufp, value, length); @@ -185,34 +185,34 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) TransformData *data = (TransformData *) TSContDataGet(contp); /* Can't TSVConnWrite() before TS_HTTP_RESPONSE_TRANSFORM_HOOK */ - if (!data->bufp) { - TSVConn connp = TSTransformOutputVConnGet(contp); + if (!data->output_bufp) { + TSVConn output_connp = TSTransformOutputVConnGet(contp); - data->bufp = TSIOBufferCreate(); - TSIOBufferReader readerp = TSIOBufferReaderAlloc(data->bufp); + data->output_bufp = TSIOBufferCreate(); + TSIOBufferReader readerp = TSIOBufferReaderAlloc(data->output_bufp); - data->viop = TSVConnWrite(connp, contp, readerp, INT64_MAX); + data->output_viop = TSVConnWrite(output_connp, contp, readerp, INT64_MAX); SHA256_Init(&data->c); } - TSVIO viop = TSVConnWriteVIOGet(contp); - TSIOBuffer bufp = TSVIOBufferGet(viop); + TSVIO input_viop = TSVConnWriteVIOGet(contp); + TSIOBuffer input_bufp = TSVIOBufferGet(input_viop); - if (!bufp) { - int ndone = TSVIONDoneGet(viop); - TSVIONBytesSet(data->viop, ndone); + if (!input_bufp) { + int ndone = TSVIONDoneGet(input_viop); + TSVIONBytesSet(data->output_viop, ndone); - TSVIOReenable(data->viop); + TSVIOReenable(data->output_viop); return 0; } - TSIOBufferReader readerp = TSVIOReaderGet(viop); + TSIOBufferReader readerp = TSVIOReaderGet(input_viop); int avail = TSIOBufferReaderAvail(readerp); if (avail > 0) { - TSIOBufferCopy(data->bufp, readerp, avail, 0); + TSIOBufferCopy(data->output_bufp, readerp, avail, 0); /* Feed content to message digest */ TSIOBufferBlock blockp = TSIOBufferReaderStart(readerp); @@ -226,27 +226,27 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) TSIOBufferReaderConsume(readerp, avail); - int ndone = TSVIONDoneGet(viop); - TSVIONDoneSet(viop, ndone + avail); + int ndone = TSVIONDoneGet(input_viop); + TSVIONDoneSet(input_viop, ndone + avail); } /* If not finished and we copied some content */ - int ntodo = TSVIONTodoGet(viop); + int ntodo = TSVIONTodoGet(input_viop); if (ntodo > 0) { if (avail > 0) { - TSContCall(TSVIOContGet(viop), TS_EVENT_VCONN_WRITE_READY, viop); + TSContCall(TSVIOContGet(input_viop), TS_EVENT_VCONN_WRITE_READY, input_viop); - TSVIOReenable(data->viop); + TSVIOReenable(data->output_viop); } /* If finished */ } else { - TSContCall(TSVIOContGet(viop), TS_EVENT_VCONN_WRITE_COMPLETE, viop); + TSContCall(TSVIOContGet(input_viop), TS_EVENT_VCONN_WRITE_COMPLETE, input_viop); - int ndone = TSVIONDoneGet(viop); - TSVIONBytesSet(data->viop, ndone); + int ndone = TSVIONDoneGet(input_viop); + TSVIONBytesSet(data->output_viop, ndone); - TSVIOReenable(data->viop); + TSVIOReenable(data->output_viop); SHA256_Final((unsigned char *) digest, &data->c); @@ -266,10 +266,10 @@ transform_vconn_write_complete(TSCont contp, void * /* edata ATS_UNUSED */) { TransformData *data = (TransformData *) TSContDataGet(contp); - TSVConn connp = TSTransformOutputVConnGet(contp); - TSVConnShutdown(connp, 0, 1); + TSVConn output_connp = TSTransformOutputVConnGet(contp); + TSVConnShutdown(output_connp, 0, 1); - TSIOBufferDestroy(data->bufp); + TSIOBufferDestroy(data->output_bufp); TSfree(data); TSContDestroy(contp); @@ -310,7 +310,7 @@ http_read_response_hdr(TSCont /* contp ATS_UNUSED */, void *edata) data->txnp = (TSHttpTxn) edata; /* Can't TSVConnWrite() before TS_HTTP_RESPONSE_TRANSFORM_HOOK */ - data->bufp = NULL; + data->output_bufp = NULL; TSVConn connp = TSTransformCreate(transform_handler, data->txnp); TSContDataSet(connp, data); From 07e418dd884a32a9b7313a38664be1161583ed07 Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Wed, 5 Feb 2014 10:07:44 -0800 Subject: [PATCH 03/16] Copy the upstream nbytes (if known) downstream to preserve the "Content-Length: ..." header. In that case there will be a TS_EVENT_VCONN_WRITE_COMPLETE event from downstream before ntodo is zero, so write the digest to the cache in transform_vconn_write_complete(). If the upstream nbytes isn't known until the end of the content, update the downstream nbytes and reenable it to get it to send a TS_EVENT_VCONN_WRITE_COMPLETE event. Fix some memory management. --- plugins/experimental/metalink/metalink.cc | 87 ++++++++++++----------- 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/plugins/experimental/metalink/metalink.cc b/plugins/experimental/metalink/metalink.cc index afb582c222f..e8fc620a5f8 100644 --- a/plugins/experimental/metalink/metalink.cc +++ b/plugins/experimental/metalink/metalink.cc @@ -124,24 +124,20 @@ cache_open_write(TSCont contp, void *edata) int length; TransformData *transform_data = (TransformData *) TSContDataGet(contp); + TSContDestroy(contp); TSCacheKeyDestroy(transform_data->key); - WriteData *write_data = (WriteData *) TSmalloc(sizeof(WriteData)); - write_data->connp = (TSVConn) edata; - - contp = TSContCreate(write_handler, NULL); - TSContDataSet(contp, write_data); - - write_data->bufp = TSIOBufferCreate(); - TSIOBufferReader readerp = TSIOBufferReaderAlloc(write_data->bufp); - if (TSHttpTxnClientReqGet(transform_data->txnp, &req_bufp, &hdr_loc) != TS_SUCCESS) { TSError("Couldn't retrieve client request header"); + TSfree(transform_data); + return 0; } + TSfree(transform_data); + if (TSHttpHdrUrlGet(req_bufp, hdr_loc, &url_loc) != TS_SUCCESS) { TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, hdr_loc); @@ -159,6 +155,15 @@ cache_open_write(TSCont contp, void *edata) TSHandleMLocRelease(req_bufp, hdr_loc, url_loc); TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, hdr_loc); + WriteData *write_data = (WriteData *) TSmalloc(sizeof(WriteData)); + write_data->connp = (TSVConn) edata; + + contp = TSContCreate(write_handler, NULL); + TSContDataSet(contp, write_data); + + write_data->bufp = TSIOBufferCreate(); + TSIOBufferReader readerp = TSIOBufferReaderAlloc(write_data->bufp); + int nbytes = TSIOBufferWrite(write_data->bufp, value, length); TSVConnWrite(write_data->connp, contp, readerp, nbytes); @@ -170,8 +175,10 @@ static int cache_open_write_failed(TSCont contp, void * /* edata ATS_UNUSED */) { TransformData *data = (TransformData *) TSContDataGet(contp); + TSContDestroy(contp); TSCacheKeyDestroy(data->key); + TSfree(data); return 0; } @@ -181,9 +188,10 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) { const char *value; int64_t length; - char digest[32]; TransformData *data = (TransformData *) TSContDataGet(contp); + TSVIO input_viop = TSVConnWriteVIOGet(contp); + /* Can't TSVConnWrite() before TS_HTTP_RESPONSE_TRANSFORM_HOOK */ if (!data->output_bufp) { TSVConn output_connp = TSTransformOutputVConnGet(contp); @@ -191,12 +199,23 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) data->output_bufp = TSIOBufferCreate(); TSIOBufferReader readerp = TSIOBufferReaderAlloc(data->output_bufp); - data->output_viop = TSVConnWrite(output_connp, contp, readerp, INT64_MAX); + int nbytes = TSVIONBytesGet(input_viop); + data->output_viop = TSVConnWrite(output_connp, contp, readerp, nbytes < 0 ? INT64_MAX : nbytes); SHA256_Init(&data->c); } - TSVIO input_viop = TSVConnWriteVIOGet(contp); + int ntodo = TSVIONTodoGet(input_viop); + if (!ntodo) { + + int ndone = TSVIONDoneGet(input_viop); + TSVIONBytesSet(data->output_viop, ndone); + + TSVIOReenable(data->output_viop); + + return 0; + } + TSIOBuffer input_bufp = TSVIOBufferGet(input_viop); if (!input_bufp) { @@ -211,7 +230,7 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) TSIOBufferReader readerp = TSVIOReaderGet(input_viop); int avail = TSIOBufferReaderAvail(readerp); - if (avail > 0) { + if (avail) { TSIOBufferCopy(data->output_bufp, readerp, avail, 0); /* Feed content to message digest */ @@ -228,34 +247,10 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) int ndone = TSVIONDoneGet(input_viop); TSVIONDoneSet(input_viop, ndone + avail); - } - - /* If not finished and we copied some content */ - int ntodo = TSVIONTodoGet(input_viop); - - if (ntodo > 0) { - if (avail > 0) { - TSContCall(TSVIOContGet(input_viop), TS_EVENT_VCONN_WRITE_READY, input_viop); - - TSVIOReenable(data->output_viop); - } - /* If finished */ - } else { - TSContCall(TSVIOContGet(input_viop), TS_EVENT_VCONN_WRITE_COMPLETE, input_viop); - - int ndone = TSVIONDoneGet(input_viop); - TSVIONBytesSet(data->output_viop, ndone); TSVIOReenable(data->output_viop); - SHA256_Final((unsigned char *) digest, &data->c); - - data->key = TSCacheKeyCreate(); - if (TSCacheKeyDigestSet(data->key, digest, sizeof(digest)) != TS_SUCCESS) { - return 0; - } - - TSCacheWrite(contp, data->key); + TSContCall(TSVIOContGet(input_viop), TS_EVENT_VCONN_WRITE_READY, input_viop); } return 0; @@ -264,15 +259,27 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) static int transform_vconn_write_complete(TSCont contp, void * /* edata ATS_UNUSED */) { + char digest[32]; + TransformData *data = (TransformData *) TSContDataGet(contp); TSVConn output_connp = TSTransformOutputVConnGet(contp); TSVConnShutdown(output_connp, 0, 1); TSIOBufferDestroy(data->output_bufp); - TSfree(data); - TSContDestroy(contp); + SHA256_Final((unsigned char *) digest, &data->c); + + data->key = TSCacheKeyCreate(); + if (TSCacheKeyDigestSet(data->key, digest, sizeof(digest)) != TS_SUCCESS) { + TSContDestroy(contp); + + TSfree(data); + + return 0; + } + + TSCacheWrite(contp, data->key); return 0; } From 4b2a7ddd8d6603361199daaa0e7f20e7127205ae Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Wed, 5 Feb 2014 10:16:59 -0800 Subject: [PATCH 04/16] Avoid segfault in TSVIOReenable() if the client disconnected --- plugins/experimental/metalink/metalink.cc | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/plugins/experimental/metalink/metalink.cc b/plugins/experimental/metalink/metalink.cc index e8fc620a5f8..170f8cfe46a 100644 --- a/plugins/experimental/metalink/metalink.cc +++ b/plugins/experimental/metalink/metalink.cc @@ -219,10 +219,19 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) TSIOBuffer input_bufp = TSVIOBufferGet(input_viop); if (!input_bufp) { - int ndone = TSVIONDoneGet(input_viop); - TSVIONBytesSet(data->output_viop, ndone); + if (TSVConnClosedGet(contp)) { + TSContDestroy(contp); - TSVIOReenable(data->output_viop); + TSIOBufferDestroy(data->output_bufp); + TSfree(data); + + } else { + + int ndone = TSVIONDoneGet(input_viop); + TSVIONBytesSet(data->output_viop, ndone); + + TSVIOReenable(data->output_viop); + } return 0; } From 4f736761fa939f9ec376f225e37a81f6cd8b4b0d Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Wed, 5 Feb 2014 10:18:54 -0800 Subject: [PATCH 05/16] Drop TSVIOBufferGet() and test TSVIOReaderGet() instead --- plugins/experimental/metalink/metalink.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/experimental/metalink/metalink.cc b/plugins/experimental/metalink/metalink.cc index 170f8cfe46a..88fc8b945ac 100644 --- a/plugins/experimental/metalink/metalink.cc +++ b/plugins/experimental/metalink/metalink.cc @@ -216,9 +216,8 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) return 0; } - TSIOBuffer input_bufp = TSVIOBufferGet(input_viop); - - if (!input_bufp) { + TSIOBufferReader readerp = TSVIOReaderGet(input_viop); + if (!readerp) { if (TSVConnClosedGet(contp)) { TSContDestroy(contp); @@ -236,7 +235,6 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) return 0; } - TSIOBufferReader readerp = TSVIOReaderGet(input_viop); int avail = TSIOBufferReaderAvail(readerp); if (avail) { From d43b8018f3d036bcb0f4b7cdb8beaf06e0ae8103 Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Wed, 5 Feb 2014 10:19:52 -0800 Subject: [PATCH 06/16] I don't know what this does --- plugins/experimental/metalink/metalink.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/experimental/metalink/metalink.cc b/plugins/experimental/metalink/metalink.cc index 88fc8b945ac..b957dead6c5 100644 --- a/plugins/experimental/metalink/metalink.cc +++ b/plugins/experimental/metalink/metalink.cc @@ -270,9 +270,6 @@ transform_vconn_write_complete(TSCont contp, void * /* edata ATS_UNUSED */) TransformData *data = (TransformData *) TSContDataGet(contp); - TSVConn output_connp = TSTransformOutputVConnGet(contp); - TSVConnShutdown(output_connp, 0, 1); - TSIOBufferDestroy(data->output_bufp); SHA256_Final((unsigned char *) digest, &data->c); From 85697757ff54f702351b990f84c9669bf19cc5c7 Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Wed, 5 Feb 2014 11:01:40 -0800 Subject: [PATCH 07/16] Improve comments --- plugins/experimental/metalink/metalink.cc | 132 +++++++++++++++++++--- 1 file changed, 114 insertions(+), 18 deletions(-) diff --git a/plugins/experimental/metalink/metalink.cc b/plugins/experimental/metalink/metalink.cc index b957dead6c5..a098e1b1ef2 100644 --- a/plugins/experimental/metalink/metalink.cc +++ b/plugins/experimental/metalink/metalink.cc @@ -38,12 +38,30 @@ #include "ts/ts.h" #include "ink_defs.h" +/* Implement TS_HTTP_READ_RESPONSE_HDR_HOOK to implement a null transform. + * Compute the SHA-256 digest of the content, write it to the cache and store + * the request URL at that key. + * + * Implement TS_HTTP_SEND_RESPONSE_HDR_HOOK to check the "Location: ..." and + * "Digest: SHA-256=..." headers. Use TSCacheRead() to check if the URL in the + * "Location: ..." header is already cached. If not, potentially rewrite that + * header. Do this after responses are cached because the cache will change. + * + * More details are on the [wiki page] in the Traffic Server wiki. + * + * [wiki page] https://cwiki.apache.org/confluence/display/TS/Metalink */ + +/* TSVConnWrite() data: Store the request URL */ + typedef struct { TSVConn connp; TSIOBuffer bufp; } WriteData; +/* TSTransformCreate() and TSCacheWrite() data: Compute the SHA-256 digest of + * the content and write it to the cache */ + typedef struct { TSHttpTxn txnp; @@ -58,6 +76,9 @@ typedef struct { } TransformData; +/* TSCacheRead() and TSVConnRead() data: Check the "Location: ..." and + * "Digest: SHA-256=..." headers */ + typedef struct { TSHttpTxn txnp; @@ -81,6 +102,10 @@ typedef struct { } SendData; +/* Implement TS_HTTP_READ_RESPONSE_HDR_HOOK to implement a null transform */ + +/* Store the request URL */ + static int write_vconn_write_complete(TSCont contp, void * /* edata ATS_UNUSED */) { @@ -88,7 +113,7 @@ write_vconn_write_complete(TSCont contp, void * /* edata ATS_UNUSED */) TSContDestroy(contp); /* The object is not committed to the cache until the vconnection is closed. - * When all data has been transferred, the user (contp) must do a + * When all the data has been transferred, the user (contp) must do a * TSVConnClose() */ TSVConnClose(data->connp); @@ -98,6 +123,8 @@ write_vconn_write_complete(TSCont contp, void * /* edata ATS_UNUSED */) return 0; } +/* TSVConnWrite() handler: Store the request URL */ + static int write_handler(TSCont contp, TSEvent event, void *edata) { @@ -112,6 +139,9 @@ write_handler(TSCont contp, TSEvent event, void *edata) return 0; } +/* Compute the SHA-256 digest of the content, write it to the cache and store + * the request URL at that key */ + static int cache_open_write(TSCont contp, void *edata) { @@ -155,9 +185,13 @@ cache_open_write(TSCont contp, void *edata) TSHandleMLocRelease(req_bufp, hdr_loc, url_loc); TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, hdr_loc); + /* Store the request URL */ + WriteData *write_data = (WriteData *) TSmalloc(sizeof(WriteData)); write_data->connp = (TSVConn) edata; + /* Can't reuse TSTransformCreate() continuation because it already implements + * TS_EVENT_VCONN_WRITE_COMPLETE */ contp = TSContCreate(write_handler, NULL); TSContDataSet(contp, write_data); @@ -171,6 +205,8 @@ cache_open_write(TSCont contp, void *edata) return 0; } +/* Do nothing */ + static int cache_open_write_failed(TSCont contp, void * /* edata ATS_UNUSED */) { @@ -183,6 +219,9 @@ cache_open_write_failed(TSCont contp, void * /* edata ATS_UNUSED */) return 0; } +/* Copy content from the input buffer to the output buffer without modification + * while at the same time feeding it to the message digest */ + static int vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) { @@ -192,19 +231,35 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) TSVIO input_viop = TSVConnWriteVIOGet(contp); - /* Can't TSVConnWrite() before TS_HTTP_RESPONSE_TRANSFORM_HOOK */ + /* Initialize data here because can't call TSVConnWrite() before + * TS_HTTP_RESPONSE_TRANSFORM_HOOK */ if (!data->output_bufp) { TSVConn output_connp = TSTransformOutputVConnGet(contp); data->output_bufp = TSIOBufferCreate(); TSIOBufferReader readerp = TSIOBufferReaderAlloc(data->output_bufp); + /* Determines the "Content-Length: ..." header + * (or "Transfer-Encoding: chunked") */ + + /* Avoid failed assert "nbytes >= 0" if "Transfer-Encoding: chunked" */ int nbytes = TSVIONBytesGet(input_viop); data->output_viop = TSVConnWrite(output_connp, contp, readerp, nbytes < 0 ? INT64_MAX : nbytes); SHA256_Init(&data->c); } + /* If the response has a "Content-Length: ..." header then ntodo will never + * be zero because there will instead be a TS_EVENT_VCONN_WRITE_COMPLETE + * event from downstream after nbytes of content. + * + * Otherwise (if the response is "Transfer-Encoding: chunked") ntodo will be + * zero when the upstream nbytes is known at the end of the content, because + * there won't be a TS_EVENT_VCONN_WRITE_COMPLETE event while the downstream + * nbytes is INT64_MAX. + * + * In that case to get it to send a TS_EVENT_VCONN_WRITE_COMPLETE event, + * update the downstream nbytes and reenable it. */ int ntodo = TSVIONTodoGet(input_viop); if (!ntodo) { @@ -216,8 +271,13 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) return 0; } + /* Avoid failed assert "sdk_sanity_check_iocore_structure(readerp) == + * TS_SUCCESS" in TSIOBufferReaderAvail() if the status code is 302? or the + * message body is empty? */ TSIOBufferReader readerp = TSVIOReaderGet(input_viop); if (!readerp) { + + /* Avoid segfault in TSVIOReenable() if the client disconnected */ if (TSVConnClosedGet(contp)) { TSContDestroy(contp); @@ -240,7 +300,7 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) if (avail) { TSIOBufferCopy(data->output_bufp, readerp, avail, 0); - /* Feed content to message digest */ + /* Feed content to the message digest */ TSIOBufferBlock blockp = TSIOBufferReaderStart(readerp); while (blockp) { @@ -252,6 +312,7 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) TSIOBufferReaderConsume(readerp, avail); + /* Call TSVIONDoneSet() for TSVIONTodoGet() condition */ int ndone = TSVIONDoneGet(input_viop); TSVIONDoneSet(input_viop, ndone + avail); @@ -263,6 +324,8 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) return 0; } +/* Write the digest to the cache */ + static int transform_vconn_write_complete(TSCont contp, void * /* edata ATS_UNUSED */) { @@ -283,11 +346,14 @@ transform_vconn_write_complete(TSCont contp, void * /* edata ATS_UNUSED */) return 0; } + /* Reuse the TSTransformCreate() continuation */ TSCacheWrite(contp, data->key); return 0; } +/* TSTransformCreate() and TSCacheWrite() handler */ + static int transform_handler(TSCont contp, TSEvent event, void *edata) { @@ -312,7 +378,8 @@ transform_handler(TSCont contp, TSEvent event, void *edata) return 0; } -/* Compute SHA-256 digest, write to cache, and store there the request URL */ +/* Compute the SHA-256 digest of the content, write it to the cache and store + * the request URL at that key */ static int http_read_response_hdr(TSCont /* contp ATS_UNUSED */, void *edata) @@ -320,7 +387,8 @@ http_read_response_hdr(TSCont /* contp ATS_UNUSED */, void *edata) TransformData *data = (TransformData *) TSmalloc(sizeof(TransformData)); data->txnp = (TSHttpTxn) edata; - /* Can't TSVConnWrite() before TS_HTTP_RESPONSE_TRANSFORM_HOOK */ + /* Can't initialize data here because can't call TSVConnWrite() before + * TS_HTTP_RESPONSE_TRANSFORM_HOOK */ data->output_bufp = NULL; TSVConn connp = TSTransformCreate(transform_handler, data->txnp); @@ -333,6 +401,11 @@ http_read_response_hdr(TSCont /* contp ATS_UNUSED */, void *edata) return 0; } +/* Implement TS_HTTP_SEND_RESPONSE_HDR_HOOK to check the "Location: ..." and + * "Digest: SHA-256=..." headers */ + +/* Read the URL stored at the digest */ + static int cache_open_read(TSCont contp, void *edata) { @@ -340,11 +413,15 @@ cache_open_read(TSCont contp, void *edata) TSVConn connp = (TSVConn) edata; data->read_bufp = TSIOBufferCreate(); + + /* Reuse the TSCacheRead() continuation */ TSVConnRead(connp, contp, data->read_bufp, INT64_MAX); return 0; } +/* Do nothing, just reenable the response */ + static int cache_open_read_failed(TSCont contp, void * /* edata ATS_UNUSED */) { @@ -363,6 +440,8 @@ cache_open_read_failed(TSCont contp, void * /* edata ATS_UNUSED */) return 0; } +/* TSCacheRead() handler: Check if the URL stored at the digest is cached */ + static int rewrite_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) { @@ -374,13 +453,14 @@ rewrite_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) switch (event) { - /* Yes: Rewrite "Location: ..." header and reenable response */ + /* Yes: Rewrite the "Location: ..." header and reenable the response */ case TS_EVENT_CACHE_OPEN_READ: value = TSUrlStringGet(data->resp_bufp, data->url_loc, &length); TSMimeHdrFieldValuesClear(data->resp_bufp, data->hdr_loc, data->location_loc); TSMimeHdrFieldValueStringInsert(data->resp_bufp, data->hdr_loc, data->location_loc, -1, value, length); break; + /* No: Do nothing, just reenable the response */ case TS_EVENT_CACHE_OPEN_READ_FAILED: break; @@ -400,6 +480,8 @@ rewrite_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) return 0; } +/* Read the URL stored at the digest */ + static int vconn_read_ready(TSCont contp, void * /* edata ATS_UNUSED */) { @@ -443,6 +525,8 @@ vconn_read_ready(TSCont contp, void * /* edata ATS_UNUSED */) return 0; } + /* Check if the URL stored at the digest is cached */ + contp = TSContCreate(rewrite_handler, NULL); TSContDataSet(contp, data); @@ -451,15 +535,19 @@ vconn_read_ready(TSCont contp, void * /* edata ATS_UNUSED */) return 0; } -/* Check if "Digest: SHA-256=..." digest already exist in cache */ +/* TSCacheRead() and TSVConnRead() handler: Check if the "Digest: SHA-256=..." + * digest already exists in the cache */ static int digest_handler(TSCont contp, TSEvent event, void *edata) { switch (event) { + + /* Yes: Read the URL stored at that key */ case TS_EVENT_CACHE_OPEN_READ: return cache_open_read(contp, edata); + /* No: Do nothing, just reenable the response */ case TS_EVENT_CACHE_OPEN_READ_FAILED: return cache_open_read_failed(contp, edata); @@ -473,7 +561,7 @@ digest_handler(TSCont contp, TSEvent event, void *edata) return 0; } -/* Check if "Location: ..." URL already exist in cache */ +/* TSCacheRead() handler: Check if the "Location: ..." URL is already cached */ static int location_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) @@ -482,11 +570,11 @@ location_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) TSContDestroy(contp); switch (event) { - /* Yes: Do nothing, just reenable response */ + /* Yes: Do nothing, just reenable the response */ case TS_EVENT_CACHE_OPEN_READ: break; - /* No: Check "Digest: SHA-256=..." digest */ + /* No: Check if the "Digest: SHA-256=..." digest already exists in the cache */ case TS_EVENT_CACHE_OPEN_READ_FAILED: { const char *value; @@ -529,6 +617,10 @@ location_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) return 0; } +/* Use TSCacheRead() to check if the URL in the "Location: ..." header is + * already cached. If not, potentially rewrite that header. Do this after + * responses are cached because the cache will change. */ + static int http_send_response_hdr(TSCont contp, void *edata) { @@ -555,12 +647,12 @@ http_send_response_hdr(TSCont contp, void *edata) /* Assumption: Want to minimize cache read, so check first that: * - * 1. response has "Location: ..." header - * 2. response has "Digest: SHA-256=..." header + * 1. response has a "Location: ..." header + * 2. response has a "Digest: SHA-256=..." header * - * Then scan if URL or digest already exist in cache */ + * Then scan if the URL or digest already exist in the cache */ - /* If response has "Location: ..." header */ + /* If the response has a "Location: ..." header */ data->location_loc = TSMimeHdrFieldFind(data->resp_bufp, data->hdr_loc, TS_MIME_FIELD_LOCATION, TS_MIME_LEN_LOCATION); if (!data->location_loc) { TSHandleMLocRelease(data->resp_bufp, TS_NULL_MLOC, data->hdr_loc); @@ -573,9 +665,9 @@ http_send_response_hdr(TSCont contp, void *edata) TSUrlCreate(data->resp_bufp, &data->url_loc); - /* If can't parse or lookup "Location: ..." URL, should still check if - * response has "Digest: SHA-256=..." header? No: Can't parse or lookup URL - * in "Location: ..." header is error */ + /* If can't parse or lookup the "Location: ..." URL, should still check if + * the response has a "Digest: SHA-256=..." header? No: Can't parse or + * lookup the URL in the "Location: ..." header is an error. */ value = TSMimeHdrFieldValueStringGet(data->resp_bufp, data->hdr_loc, data->location_loc, -1, &length); if (TSUrlParse(data->resp_bufp, data->url_loc, &value, value + length) != TS_PARSE_DONE) { @@ -603,7 +695,7 @@ http_send_response_hdr(TSCont contp, void *edata) return 0; } - /* ... and "Digest: SHA-256=..." header */ + /* ... and a "Digest: SHA-256=..." header */ data->digest_loc = TSMimeHdrFieldFind(data->resp_bufp, data->hdr_loc, "Digest", 6); while (data->digest_loc) { @@ -615,6 +707,8 @@ http_send_response_hdr(TSCont contp, void *edata) continue; } + /* Check if the "Location: ..." URL is already cached */ + contp = TSContCreate(location_handler, NULL); TSContDataSet(contp, data); @@ -630,6 +724,8 @@ http_send_response_hdr(TSCont contp, void *edata) data->digest_loc = next_loc; } + /* Didn't find a "Digest: SHA-256=..." header, just reenable the response */ + TSCacheKeyDestroy(data->key); TSHandleMLocRelease(data->resp_bufp, TS_NULL_MLOC, data->url_loc); From 274a45f13d11c52374f692222cdaf8f9aa3635db Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Wed, 5 Feb 2014 11:06:50 -0800 Subject: [PATCH 08/16] Minor edits --- plugins/experimental/metalink/README | 33 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/plugins/experimental/metalink/README b/plugins/experimental/metalink/README index e56afaab632..7f56dd81ff5 100644 --- a/plugins/experimental/metalink/README +++ b/plugins/experimental/metalink/README @@ -6,7 +6,7 @@ Take standard headers and knowledge about objects in the cache and potentially rewrite those headers so that a client will use a URL - that is already cached instead of one that isn't. The headers are + that's already cached instead of one that isn't. The headers are specified in [RFC 6429] (Metalink/HTTP: Mirrors and Hashes) and [RFC 3230] (Instance Digests in HTTP) and are sent by various download redirectors or content distribution networks. @@ -21,7 +21,7 @@ different mirrors and users don't know which mirrors are already cached. These sites often present users with a simple download button, but the button doesn't predictably access the same mirror, - or a mirror that is already cached. To users it seems like the + or a mirror that's already cached. To users it seems like the download works sometimes (takes seconds) and not others (takes hours), which is frustrating. @@ -31,31 +31,31 @@ [How to cache openSUSE repositories with Squid] is another, different example of a use case where picking a URL that's already - cached is important. + cached is valuable. 2. What it Does When it sees a response with a "Location: ..." header and a - "Digest: SHA-256=..." header, it checks to see if the URL in the - Location header is already cached. If it isn't, then it tries to - find a URL that is cached to use instead. It looks in the cache - for some object that matches the digest in the Digest header and if - it finds something, then it rewites the Location header with the - URL from that object. + "Digest: SHA-256=..." header, it checks if the URL in the Location + header is already cached. If it isn't, then it tries to find a URL + that is cached to use instead. It looks in the cache for some + object that matches the digest in the Digest header and if it + succeeds, then it rewites the Location header with the URL from + that object. - That way a client should get sent to a URL that's already cached - and the user won't end up downloading the file again. + This way a client should get sent to a URL that's already cached + and won't download the file again. 3. How to Use it - Just build the plugin and then add it to your plugins.config file. + Just build the plugin and add it to your plugins.config file. The code is distributed along with recent versions of Traffic - Server, in the "plugins/experimental" directory. To build it, pass - the "--enable-experimental-plugins" option to the Traffic Server - configure script when you build Traffic Server: + Server, in the "plugins/experimental/metalink" directory. To build + it, pass the "--enable-experimental-plugins" option to the Traffic + Server configure script when you build Traffic Server: 
$ ./configure --enable-experimental-plugins
@@ -65,8 +65,7 @@ 4. Read More - More details are to be found on the [wiki page] in the Traffic - Server wiki. + More details are on the [wiki page] in the Traffic Server wiki. [RFC 6429] http://tools.ietf.org/html/rfc6249 From 943364c2ba60026a7b289091deb5960ef6097580 Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Wed, 5 Feb 2014 12:08:30 -0800 Subject: [PATCH 09/16] TS-2553 CHANGES --- CHANGES | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES b/CHANGES index 8b5a3188507..c09d4ea9cc7 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,9 @@ -*- coding: utf-8 -*- Changes with Apache Traffic Server 5.0.0 + *) [TS-2553] Fix a segfault in the Metalink plugin reported by Faysal Banna + and preserve the Content-Length header + *) [TS-2195] Remove the (deprecated) TSHttpTxnCacheLookupSkip API. *) [TS-2229] Deprecate the header_filter plugin, use header_rewrite instead. From 69d38bcabf56354af2753e5df469648d66034baa Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Wed, 19 Feb 2014 14:01:47 -0800 Subject: [PATCH 10/16] TS-2578 Close the client connection when you close TransformTerminus --- CHANGES | 2 ++ proxy/Transform.cc | 16 ++++++---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/CHANGES b/CHANGES index c09d4ea9cc7..6c202499159 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,8 @@ -*- coding: utf-8 -*- Changes with Apache Traffic Server 5.0.0 + *) [TS-2578] Close the client connection when you close TransformTerminus + *) [TS-2553] Fix a segfault in the Metalink plugin reported by Faysal Banna and preserve the Content-Length header diff --git a/proxy/Transform.cc b/proxy/Transform.cc index 2da3301f68c..6ea9f992520 100644 --- a/proxy/Transform.cc +++ b/proxy/Transform.cc @@ -251,20 +251,16 @@ TransformTerminus::handle_event(int event, void * /* edata ATS_UNUSED */) // exist). if (m_tvc->m_closed == 0) { if (m_closed == TS_VC_CLOSE_ABORT) { - if (m_read_vio.op == VIO::NONE) { - if (!m_called_user) { - m_called_user = 1; - m_tvc->m_cont->handleEvent(VC_EVENT_ERROR, NULL); - } + if (m_read_vio.op == VIO::NONE && !m_called_user) { + m_called_user = 1; + m_tvc->m_cont->handleEvent(VC_EVENT_ERROR, NULL); } else { m_read_vio._cont->handleEvent(VC_EVENT_ERROR, &m_read_vio); } } else { - if (m_read_vio.op == VIO::NONE) { - if (!m_called_user) { - m_called_user = 1; - m_tvc->m_cont->handleEvent(VC_EVENT_EOS, NULL); - } + if (m_read_vio.op == VIO::NONE && !m_called_user) { + m_called_user = 1; + m_tvc->m_cont->handleEvent(VC_EVENT_EOS, NULL); } else { m_read_vio._cont->handleEvent(VC_EVENT_EOS, &m_read_vio); } From 3ff556f2a1c12c3ec85fce76f5071d492be31ef6 Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Thu, 20 Feb 2014 13:00:30 -0800 Subject: [PATCH 11/16] metalink: If not at the end yet and can't read any more content then can't compute the digest --- plugins/experimental/metalink/metalink.cc | 29 ++++++++++------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/plugins/experimental/metalink/metalink.cc b/plugins/experimental/metalink/metalink.cc index a098e1b1ef2..63e970e64b6 100644 --- a/plugins/experimental/metalink/metalink.cc +++ b/plugins/experimental/metalink/metalink.cc @@ -272,25 +272,22 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) } /* Avoid failed assert "sdk_sanity_check_iocore_structure(readerp) == - * TS_SUCCESS" in TSIOBufferReaderAvail() if the status code is 302? or the - * message body is empty? */ + * TS_SUCCESS" in TSIOBufferReaderAvail() if the client or server disconnects + * or the content length is zero. + * + * Don't update the downstream nbytes and reenable it because if not at the + * end yet and can't read any more content then can't compute the digest. + * + * (There hasn't been a TS_EVENT_VCONN_WRITE_COMPLETE event from downstream + * yet so if the response has a "Content-Length: ..." header, it is greater + * than the content so far. ntodo is still greater than zero so if the + * response is "Transfer-Encoding: chunked", not at the end yet.) */ TSIOBufferReader readerp = TSVIOReaderGet(input_viop); if (!readerp) { + TSContDestroy(contp); - /* Avoid segfault in TSVIOReenable() if the client disconnected */ - if (TSVConnClosedGet(contp)) { - TSContDestroy(contp); - - TSIOBufferDestroy(data->output_bufp); - TSfree(data); - - } else { - - int ndone = TSVIONDoneGet(input_viop); - TSVIONBytesSet(data->output_viop, ndone); - - TSVIOReenable(data->output_viop); - } + TSIOBufferDestroy(data->output_bufp); + TSfree(data); return 0; } From 778bbb11309f2008cb338bbd0c19634de66fdae9 Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Thu, 20 Feb 2014 13:06:53 -0800 Subject: [PATCH 12/16] metalink: Zero the downstream nbytes is a shortcut to get it to send a TS_EVENT_VCONN_WRITE_COMPLETE event --- plugins/experimental/metalink/metalink.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/experimental/metalink/metalink.cc b/plugins/experimental/metalink/metalink.cc index 63e970e64b6..a775bd96a72 100644 --- a/plugins/experimental/metalink/metalink.cc +++ b/plugins/experimental/metalink/metalink.cc @@ -259,12 +259,11 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) * nbytes is INT64_MAX. * * In that case to get it to send a TS_EVENT_VCONN_WRITE_COMPLETE event, - * update the downstream nbytes and reenable it. */ + * update the downstream nbytes and reenable it. Zero the downstream nbytes + * is a shortcut. */ int ntodo = TSVIONTodoGet(input_viop); if (!ntodo) { - - int ndone = TSVIONDoneGet(input_viop); - TSVIONBytesSet(data->output_viop, ndone); + TSVIONBytesSet(data->output_viop, 0); TSVIOReenable(data->output_viop); From b5d236989d2997d8a98e39295e176ad62cba12b5 Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Thu, 20 Feb 2014 13:11:30 -0800 Subject: [PATCH 13/16] TS-2553 metalink: Avoid failed assert "sdk_sanity_check_iocore_structure(connp) == TS_SUCCESS" in TSVConnWrite() if the response is 304 Not Modified --- plugins/experimental/metalink/metalink.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/experimental/metalink/metalink.cc b/plugins/experimental/metalink/metalink.cc index a775bd96a72..65f00a7ac6c 100644 --- a/plugins/experimental/metalink/metalink.cc +++ b/plugins/experimental/metalink/metalink.cc @@ -234,7 +234,17 @@ vconn_write_ready(TSCont contp, void * /* edata ATS_UNUSED */) /* Initialize data here because can't call TSVConnWrite() before * TS_HTTP_RESPONSE_TRANSFORM_HOOK */ if (!data->output_bufp) { + + /* Avoid failed assert "sdk_sanity_check_iocore_structure(connp) == + * TS_SUCCESS" in TSVConnWrite() if the response is 304 Not Modified */ TSVConn output_connp = TSTransformOutputVConnGet(contp); + if (!output_connp) { + TSContDestroy(contp); + + TSfree(data); + + return 0; + } data->output_bufp = TSIOBufferCreate(); TSIOBufferReader readerp = TSIOBufferReaderAlloc(data->output_bufp); From f28d2180a5fcee32f11dcc693d085f3f1b1efb1f Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Thu, 20 Feb 2014 13:13:57 -0800 Subject: [PATCH 14/16] metalink: Add some functional tests --- .../metalink/test/chunkedEncoding | 81 ++++++++++++++++ .../metalink/test/chunkedEncodingDisconnect | 81 ++++++++++++++++ .../metalink/test/clientDisconnect | 78 +++++++++++++++ .../experimental/metalink/test/contentLength | 83 ++++++++++++++++ .../metalink/test/contentLengthDisconnect | 76 +++++++++++++++ .../test/finalChunkedEncodingDisconnect | 94 +++++++++++++++++++ plugins/experimental/metalink/test/http09 | 68 ++++++++++++++ plugins/experimental/metalink/test/longer | 76 +++++++++++++++ .../experimental/metalink/test/notModified | 61 ++++++++++++ .../test/shortChunkedEncodingDisconnect | 80 ++++++++++++++++ .../metalink/test/shortClientDisconnect | 74 +++++++++++++++ .../test/shortContentLengthDisconnect | 77 +++++++++++++++ plugins/experimental/metalink/test/zero | 74 +++++++++++++++ 13 files changed, 1003 insertions(+) create mode 100755 plugins/experimental/metalink/test/chunkedEncoding create mode 100755 plugins/experimental/metalink/test/chunkedEncodingDisconnect create mode 100755 plugins/experimental/metalink/test/clientDisconnect create mode 100755 plugins/experimental/metalink/test/contentLength create mode 100755 plugins/experimental/metalink/test/contentLengthDisconnect create mode 100755 plugins/experimental/metalink/test/finalChunkedEncodingDisconnect create mode 100755 plugins/experimental/metalink/test/http09 create mode 100755 plugins/experimental/metalink/test/longer create mode 100755 plugins/experimental/metalink/test/notModified create mode 100755 plugins/experimental/metalink/test/shortChunkedEncodingDisconnect create mode 100755 plugins/experimental/metalink/test/shortClientDisconnect create mode 100755 plugins/experimental/metalink/test/shortContentLengthDisconnect create mode 100755 plugins/experimental/metalink/test/zero diff --git a/plugins/experimental/metalink/test/chunkedEncoding b/plugins/experimental/metalink/test/chunkedEncoding new file mode 100755 index 00000000000..7bea156eba3 --- /dev/null +++ b/plugins/experimental/metalink/test/chunkedEncoding @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +print '''1..1 chunkedEncoding +# The proxy forwards the final chunk at the end of a chunked response''' + +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + +def callback(): + print 'not ok 1 - No final chunk yet' + + reactor.stop() + +reactor.callLater(2, callback) + +class factory(http.HTTPFactory): + class protocol(http.HTTPChannel): + class requestFactory(http.Request): + def requestReceived(ctx, method, target, version): + + ctx.client = None + ctx.clientproto = version + + ctx.write('chunkedEncoding') + + # If the proxy reads the final chunk before it sends the response + # headers, it may send a Content-Length header vs. a chunked response + reactor.callLater(1, ctx.finish) + +server = tcp.Port(0, factory()) +server.startListening() + +print '# Listening on {0}:{1}'.format(*server.socket.getsockname()) + +class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + + print 'Bail out!' + reason.printTraceback() + + reactor.stop() + + class protocol(http.HTTPClient): + def connectionLost(ctx, reason): + try: + reactor.stop() + + except error.ReactorNotRunning: + pass + + else: + print 'not ok 1 - Did the proxy crash? (The client connection closed.)' + + connectionMade = lambda ctx: ctx.transport.write('GET {0}:{1} HTTP/1.1\r\n\r\n'.format(*server.socket.getsockname())) + + def handleHeader(ctx, k, v): + if k.lower() == 'content-length': + print 'not ok 1 - Got a Content-Length header vs. a chunked response' + + # No hope of a final chunk now + reactor.stop() + + # Avoid calling undefined handleResponse() at the end of the content (if + # the proxy sent a Content-Length header vs. a chunked response). + # (Override connectionLost() when the proxy crashes or stop the reactor.) + # + # The data that was already received will be processed (the end of the + # headers), then shutdown events will fire (connections will be closed), + # and then finally the reactor will grind to a halt. + def handleResponseEnd(ctx): + pass + + def handleResponsePart(ctx, data): + if data.endswith('0\r\n\r\n'): + print 'ok 1 - Got the final chunk' + + reactor.stop() + +tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect() + +reactor.run() diff --git a/plugins/experimental/metalink/test/chunkedEncodingDisconnect b/plugins/experimental/metalink/test/chunkedEncodingDisconnect new file mode 100755 index 00000000000..c9ad0555136 --- /dev/null +++ b/plugins/experimental/metalink/test/chunkedEncodingDisconnect @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +print '''1..1 chunkedEncodingDisconnect +# The proxy closes the client connection and doesn't send a final chunk if the +# server disconnects without sending one''' + +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + +def callback(): + print 'not ok 1 - The client was left hanging' + + reactor.stop() + +reactor.callLater(2, callback) + +class factory(http.HTTPFactory): + class protocol(http.HTTPChannel): + class requestFactory(http.Request): + def requestReceived(ctx, method, target, version): + + ctx.client = None + ctx.clientproto = version + + ctx.write('chunkedEncodingDisconnect') + + # If the server disconnects before the proxy sends the response + # headers, the proxy may send a Content-Length header vs. a chunked + # response + reactor.callLater(1, ctx.transport.loseConnection) + +server = tcp.Port(0, factory()) +server.startListening() + +print '# Listening on {0}:{1}'.format(*server.socket.getsockname()) + +class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + + print 'Bail out!' + reason.printTraceback() + + reactor.stop() + + class protocol(http.HTTPClient): + def connectionLost(ctx, reason): + try: + reactor.stop() + + except error.ReactorNotRunning: + pass + + else: + print 'ok 1 - The client connection closed' + + connectionMade = lambda ctx: ctx.transport.write('GET {0}:{1} HTTP/1.1\r\n\r\n'.format(*server.socket.getsockname())) + + def handleHeader(ctx, k, v): + if k.lower() == 'content-length': + print 'not ok 1 - Got a Content-Length header vs. a chunked response' + + # Who cares what happens now? + reactor.stop() + + # Avoid calling undefined handleResponse() at the end of the content (if + # the proxy sent a Content-Length header vs. a chunked response). + # (Override connectionLost() when the proxy closes the client connection or + # stop the reactor.) + def handleResponseEnd(ctx): + pass + + def handleResponsePart(ctx, data): + if data.endswith('0\r\n\r\n'): + print 'not ok 1 - Got a final chunk' + + # Who cares what happens now? + reactor.stop() + +tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect() + +reactor.run() diff --git a/plugins/experimental/metalink/test/clientDisconnect b/plugins/experimental/metalink/test/clientDisconnect new file mode 100755 index 00000000000..a5ffadb7fc8 --- /dev/null +++ b/plugins/experimental/metalink/test/clientDisconnect @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +print '''1..1 clientDissconnect +# The proxy doesn't crash if the client disconnects prematurely''' + +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + +def callback(): + print 'not ok 1 - Why didn\'t the test finish yet?' + + reactor.stop() + +reactor.callLater(3, callback) + +class factory(http.HTTPFactory): + class protocol(http.HTTPChannel): + class requestFactory(http.Request): + def requestReceived(ctx, method, target, version): + + ctx.client = None + ctx.clientproto = version + + ctx.write('clientDisconnect') + + # The proxy crashes only after the response is complete + def callback(): + try: + ctx.finish() + + except RuntimeError: + pass + + # Open another connection + class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + print 'not ok 1 - Did the proxy crash? (Can\'t open another connection to it.)' + + reactor.stop() + + class protocol(protocol.Protocol): + def connectionMade(ctx): + print 'ok 1 - The proxy didn\'t crash (opened another connection to it)' + + reactor.stop() + + reactor.callLater(1, tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect) + + reactor.callLater(1, callback) + +server = tcp.Port(0, factory()) +server.startListening() + +print '# Listening on {0}:{1}'.format(*server.socket.getsockname()) + +class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + + print 'Bail out!' + reason.printTraceback() + + reactor.stop() + + class protocol(http.HTTPClient): + def connectionMade(ctx): + ctx.transport.write('GET {0}:{1} HTTP/1.1\r\n\r\n'.format(*server.socket.getsockname())) + + # Disconnect after the proxy sends the response headers + reactor.callLater(1, ctx.transport.loseConnection) + + # Avoid calling undefined handleResponse() at the end of the content or + # when the connection closes + def handleResponseEnd(ctx): + pass + +tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect() + +reactor.run() diff --git a/plugins/experimental/metalink/test/contentLength b/plugins/experimental/metalink/test/contentLength new file mode 100755 index 00000000000..7bcba51b8fe --- /dev/null +++ b/plugins/experimental/metalink/test/contentLength @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +print '''1..1 contentLength +# The proxy forwards the Content-Length header to the client''' + +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + +def callback(): + print 'not ok 1 - Why didn\'t the test finish yet?' + + reactor.stop() + +reactor.callLater(1, callback) + +class factory(http.HTTPFactory): + class protocol(http.HTTPChannel): + class requestFactory(http.Request): + def requestReceived(ctx, method, target, version): + + ctx.client = None + ctx.clientproto = version + + ctx.setHeader('Content-Length', 13) + ctx.write('contentLength') + +server = tcp.Port(0, factory()) +server.startListening() + +print '# Listening on {0}:{1}'.format(*server.socket.getsockname()) + +class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + + print 'Bail out!' + reason.printTraceback() + + reactor.stop() + + class protocol(http.HTTPClient): + def connectionLost(ctx, reason): + try: + reactor.stop() + + except error.ReactorNotRunning: + pass + + else: + print 'not ok 1 - Did the proxy crash? (The client connection closed.)' + + connectionMade = lambda ctx: ctx.transport.write('GET {0}:{1} HTTP/1.1\r\n\r\n'.format(*server.socket.getsockname())) + + def handleEndHeaders(ctx): + try: + reactor.stop() + + except error.ReactorNotRunning: + pass + + else: + print 'not ok 1 - No Content-Length header' + + def handleHeader(ctx, k, v): + if k.lower() == 'content-length': + if v != '13': + print 'not', + + print 'ok 1 - Content-Length header' + + reactor.stop() + + # Avoid calling undefined handleResponse() at the end of the content. + # (Override connectionLost() when the proxy crashes or stop the reactor.) + # + # The data that was already received will be processed (the end of the + # headers), then shutdown events will fire (connections will be closed), + # and then finally the reactor will grind to a halt. + def handleResponseEnd(ctx): + pass + +tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect() + +reactor.run() diff --git a/plugins/experimental/metalink/test/contentLengthDisconnect b/plugins/experimental/metalink/test/contentLengthDisconnect new file mode 100755 index 00000000000..8ba28ef66b8 --- /dev/null +++ b/plugins/experimental/metalink/test/contentLengthDisconnect @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +print '''1..2 contentLengthDisconnect +# The proxy closes the client connection if the server disconnects prematurely''' + +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + +def callback(): + print 'not ok 2 - The client was left hanging' + + reactor.stop() + +reactor.callLater(2, callback) + +class factory(http.HTTPFactory): + class protocol(http.HTTPChannel): + class requestFactory(http.Request): + def requestReceived(ctx, method, target, version): + + ctx.client = None + ctx.clientproto = version + + ctx.setHeader('Content-Length', 24) + ctx.write('contentLengthDisconnect') + + # If the server disconnects before the proxy sends the response + # headers, the proxy may send the wrong Content-Length header + reactor.callLater(1, ctx.transport.loseConnection) + +server = tcp.Port(0, factory()) +server.startListening() + +print '# Listening on {0}:{1}'.format(*server.socket.getsockname()) + +class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + + print 'Bail out!' + reason.printTraceback() + + reactor.stop() + + class protocol(http.HTTPClient): + def connectionLost(ctx, reason): + try: + reactor.stop() + + except error.ReactorNotRunning: + pass + + else: + print 'ok 2 - The client connection closed' + + connectionMade = lambda ctx: ctx.transport.write('GET {0}:{1} HTTP/1.1\r\n\r\n'.format(*server.socket.getsockname())) + + def handleHeader(ctx, k, v): + if k.lower() == 'content-length': + if v != '24': + print 'not', + + # Who cares what happens now? + reactor.stop() + + print 'ok 1 - Content-Length header' + + # Avoid calling undefined handleResponse() at the end of the content (if + # the proxy sent the wrong Content-Length header). (Override + # connectionLost() when the proxy closes the client connection or stop the + # reactor.) + def handleResponseEnd(ctx): + pass + +tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect() + +reactor.run() diff --git a/plugins/experimental/metalink/test/finalChunkedEncodingDisconnect b/plugins/experimental/metalink/test/finalChunkedEncodingDisconnect new file mode 100755 index 00000000000..6b6b7d31f54 --- /dev/null +++ b/plugins/experimental/metalink/test/finalChunkedEncodingDisconnect @@ -0,0 +1,94 @@ +#!/usr/bin/env python + +print '''1..1 finalChunkEncodingDisconnect +# The proxy forwards the final chunk even if the server disconnects immediately +# afterward''' + +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + +def callback(): + print 'not ok 1 - No final chunk yet' + + reactor.stop() + +reactor.callLater(2, callback) + +class factory(http.HTTPFactory): + class protocol(http.HTTPChannel): + class requestFactory(http.Request): + def requestReceived(ctx, method, target, version): + + ctx.client = None + ctx.clientproto = version + + ctx.write('finalChunkedEncodingDisconnect') + + # If the proxy reads the final chunk before it sends the response + # headers, it may send a Content-Length header vs. a chunked response + def callback(): + try: + ctx.finish() + + except RuntimeError: + print 'not ok 1 - Did the proxy crash? (The server connection closed.)' + + reactor.stop() + + else: + ctx.transport.loseConnection() + + reactor.callLater(1, callback) + +server = tcp.Port(0, factory()) +server.startListening() + +print '# Listening on {0}:{1}'.format(*server.socket.getsockname()) + +class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + + print 'Bail out!' + reason.printTraceback() + + reactor.stop() + + class protocol(http.HTTPClient): + def connectionLost(ctx, reason): + try: + reactor.stop() + + except error.ReactorNotRunning: + pass + + else: + print 'not ok 1 - Did the proxy crash? (The client connection closed.)' + + connectionMade = lambda ctx: ctx.transport.write('GET {0}:{1} HTTP/1.1\r\n\r\n'.format(*server.socket.getsockname())) + + def handleHeader(ctx, k, v): + if k.lower() == 'content-length': + print 'not ok 1 - Got a Content-Length header vs. a chunked response' + + # No hope of a final chunk now + reactor.stop() + + # Avoid calling undefined handleResponse() at the end of the content (if + # the proxy sent a Content-Length header vs. a chunked response). + # (Override connectionLost() when the proxy crashes or stop the reactor.) + # + # The data that was already received will be processed (the end of the + # headers), then shutdown events will fire (connections will be closed), + # and then finally the reactor will grind to a halt. + def handleResponseEnd(ctx): + pass + + def handleResponsePart(ctx, data): + if data.endswith('0\r\n\r\n'): + print 'ok 1 - Got the final chunk' + + reactor.stop() + +tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect() + +reactor.run() diff --git a/plugins/experimental/metalink/test/http09 b/plugins/experimental/metalink/test/http09 new file mode 100755 index 00000000000..48a44b76d03 --- /dev/null +++ b/plugins/experimental/metalink/test/http09 @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +print '''1..1 http09 +# The proxy doesn't crash on an HTTP/0.9 response''' + +# http://www.w3.org/Protocols/HTTP/AsImplemented +# +# The proxy crashes only after the response is complete. It closes the client +# connection whether it crashes or not because an HTTP/0.9 response is complete +# only after the server closes its connection, and then the proxy normally does +# the same thing to the client connection (although it upgrades the response to +# HTTP/1.1). So the only way to check that the proxy didn't crash is to open +# another connection. + +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + +def callback(): + print 'not ok 1 - Why didn\'t the test finish yet?' + + reactor.stop() + +reactor.callLater(2, callback) + +class factory(protocol.Factory): + class protocol(protocol.Protocol): + def connectionMade(ctx): + ctx.transport.write('http09\r\n') + + # The proxy crashes only after the response is complete + ctx.transport.loseConnection() + +server = tcp.Port(0, factory()) +server.startListening() + +print '# Listening on {0}:{1}'.format(*server.socket.getsockname()) + +class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + + print 'Bail out!' + reason.printTraceback() + + reactor.stop() + + class protocol(http.HTTPClient): + def connectionLost(ctx, reason): + + # Open another connection + class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + print 'not ok 1 - Did the proxy crash? (Can\'t open another connection to it.)' + + reactor.stop() + + class protocol(protocol.Protocol): + def connectionMade(ctx): + print 'ok 1 - The proxy didn\'t crash (opened another connection to it)' + + reactor.stop() + + reactor.callLater(1, tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect) + + connectionMade = lambda ctx: ctx.transport.write('GET {0}:{1} HTTP/1.1\r\n\r\n'.format(*server.socket.getsockname())) + +tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect() + +reactor.run() diff --git a/plugins/experimental/metalink/test/longer b/plugins/experimental/metalink/test/longer new file mode 100755 index 00000000000..64509b6b292 --- /dev/null +++ b/plugins/experimental/metalink/test/longer @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +print '''1..1 longer +# The proxy doesn't choke if the server sends more content than it advertised''' + +# Unlike the contentLength test, don't stop the reactor at the end of the +# headers. Give the proxy time to choke. + +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + +def callback(): + print 'not ok 1 - No Content-Length header' + + reactor.stop() + +reactor.callLater(1, callback) + +class factory(http.HTTPFactory): + class protocol(http.HTTPChannel): + class requestFactory(http.Request): + def requestReceived(ctx, method, target, version): + + ctx.client = None + ctx.clientproto = version + + ctx.setHeader('Content-Length', 1) + ctx.write('longer') + +server = tcp.Port(0, factory()) +server.startListening() + +print '# Listening on {0}:{1}'.format(*server.socket.getsockname()) + +class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + + print 'Bail out!' + reason.printTraceback() + + reactor.stop() + + class protocol(http.HTTPClient): + def connectionLost(ctx, reason): + try: + reactor.stop() + + except error.ReactorNotRunning: + pass + + else: + print 'not ok 1 - Did the proxy crash? (The client connection closed.)' + + connectionMade = lambda ctx: ctx.transport.write('GET {0}:{1} HTTP/1.1\r\n\r\n'.format(*server.socket.getsockname())) + + def handleHeader(ctx, k, v): + if k.lower() == 'content-length': + if v != '1': + print 'not', + + print 'ok 1 - Content-Length header' + + reactor.stop() + + # Avoid calling undefined handleResponse() at the end of the content. + # (Override connectionLost() when the proxy crashes or stop the reactor.) + # + # The data that was already received will be processed (the end of the + # headers), then shutdown events will fire (connections will be closed), + # and then finally the reactor will grind to a halt. + def handleResponseEnd(ctx): + pass + +tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect() + +reactor.run() diff --git a/plugins/experimental/metalink/test/notModified b/plugins/experimental/metalink/test/notModified new file mode 100755 index 00000000000..f0f256fbb31 --- /dev/null +++ b/plugins/experimental/metalink/test/notModified @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +print '''1..2 notModified +# The proxy doesn't crash on a 304 Not Modified response''' + +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + +def callback(): + print 'ok 2 - The proxy didn\'t crash (the client connection didn\'t close yet)' + + reactor.stop() + +reactor.callLater(1, callback) + +class factory(http.HTTPFactory): + class protocol(http.HTTPChannel): + class requestFactory(http.Request): + def requestReceived(ctx, method, target, version): + + ctx.client = None + ctx.clientproto = version + + ctx.setResponseCode(304) + ctx.finish() + +server = tcp.Port(0, factory()) +server.startListening() + +print '# Listening on {0}:{1}'.format(*server.socket.getsockname()) + +class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + + print 'Bail out!' + reason.printTraceback() + + reactor.stop() + + class protocol(http.HTTPClient): + def connectionLost(ctx, reason): + try: + reactor.stop() + + except error.ReactorNotRunning: + pass + + else: + print 'not ok 1 - Did the proxy crash? (The client connection closed.)' + + connectionMade = lambda ctx: ctx.transport.write('GET {0}:{1} HTTP/1.1\r\n\r\n'.format(*server.socket.getsockname())) + + def handleStatus(ctx, version, status, message): + if status != '304': + print 'not', + + print 'ok 1 - 304 Not Modified response status' + +tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect() + +reactor.run() diff --git a/plugins/experimental/metalink/test/shortChunkedEncodingDisconnect b/plugins/experimental/metalink/test/shortChunkedEncodingDisconnect new file mode 100755 index 00000000000..46487d8aa5e --- /dev/null +++ b/plugins/experimental/metalink/test/shortChunkedEncodingDisconnect @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +print '''1..1 shortChunkedEncodingDisconnect +# The proxy closes the client connection and doesn't send a final chunk if the +# server disconnects without sending one, before the proxy sends the response +# headers''' + +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + +def callback(): + print 'not ok 1 - The client was left hanging' + + reactor.stop() + +reactor.callLater(1, callback) + +class factory(http.HTTPFactory): + class protocol(http.HTTPChannel): + class requestFactory(http.Request): + def requestReceived(ctx, method, target, version): + + ctx.client = None + ctx.clientproto = version + + ctx.write('shortChunkedEncodingDisconnect') + + # Disconnect before the proxy sends the response headers + ctx.transport.loseConnection() + +server = tcp.Port(0, factory()) +server.startListening() + +print '# Listening on {0}:{1}'.format(*server.socket.getsockname()) + +class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + + print 'Bail out!' + reason.printTraceback() + + reactor.stop() + + class protocol(http.HTTPClient): + def connectionLost(ctx, reason): + try: + reactor.stop() + + except error.ReactorNotRunning: + pass + + else: + print 'ok 1 - The client connection closed' + + connectionMade = lambda ctx: ctx.transport.write('GET {0}:{1} HTTP/1.1\r\n\r\n'.format(*server.socket.getsockname())) + + def handleHeader(ctx, k, v): + if k.lower() == 'content-length': + print 'not ok 1 - Got a Content-Length header vs. a chunked response' + + # Who cares what happens now? + reactor.stop() + + # Avoid calling undefined handleResponse() at the end of the content (if + # the proxy sent a Content-Length header vs. a chunked response). + # (Override connectionLost() when the proxy closes the client connection or + # stop the reactor.) + def handleResponseEnd(ctx): + pass + + def handleResponsePart(ctx, data): + if data.endswith('0\r\n\r\n'): + print 'not ok 1 - Got a final chunk' + + # Who cares what happens now? + reactor.stop() + +tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect() + +reactor.run() diff --git a/plugins/experimental/metalink/test/shortClientDisconnect b/plugins/experimental/metalink/test/shortClientDisconnect new file mode 100755 index 00000000000..082abeb8879 --- /dev/null +++ b/plugins/experimental/metalink/test/shortClientDisconnect @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +print '''1..1 shortClientDisconnect +# The proxy doesn't crash if the client disconnects before the proxy sends the +# response headers''' + +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + +def callback(): + print 'not ok 1 - Why didn\'t the test finish yet?' + + reactor.stop() + +reactor.callLater(3, callback) + +class factory(http.HTTPFactory): + class protocol(http.HTTPChannel): + class requestFactory(http.Request): + def requestReceived(ctx, method, target, version): + + ctx.client = None + ctx.clientproto = version + + ctx.write('shortClientDisconnect0') + + def callback(): + ctx.write('shortClientDisconnect1') + + # Open another connection + class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + print 'not ok 1 - Did the proxy crash? (Can\'t open another connection to it.)' + + reactor.stop() + + class protocol(protocol.Protocol): + def connectionMade(ctx): + print 'ok 1 - The proxy didn\'t crash (opened another connection to it)' + + reactor.stop() + + reactor.callLater(1, tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect) + + reactor.callLater(1, callback) + +server = tcp.Port(0, factory()) +server.startListening() + +print '# Listening on {0}:{1}'.format(*server.socket.getsockname()) + +class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + + print 'Bail out!' + reason.printTraceback() + + reactor.stop() + + class protocol(http.HTTPClient): + def connectionMade(ctx): + ctx.transport.write('GET {0}:{1} HTTP/1.1\r\n\r\n'.format(*server.socket.getsockname())) + + # Disconnect before the proxy sends the response headers + ctx.transport.loseConnection() + + # Avoid calling undefined handleResponse() at the end of the content or + # when the connection closes + def handleResponseEnd(ctx): + pass + +tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect() + +reactor.run() diff --git a/plugins/experimental/metalink/test/shortContentLengthDisconnect b/plugins/experimental/metalink/test/shortContentLengthDisconnect new file mode 100755 index 00000000000..8dae09546cb --- /dev/null +++ b/plugins/experimental/metalink/test/shortContentLengthDisconnect @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +print '''1..2 shortContentLengthDisconnect +# The proxy sends the right Content-Length header and closes the client +# connection if the server disconnects before the proxy sends the response +# headers''' + +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + +def callback(): + print 'not ok 2 - The client was left hanging' + + reactor.stop() + +reactor.callLater(1, callback) + +class factory(http.HTTPFactory): + class protocol(http.HTTPChannel): + class requestFactory(http.Request): + def requestReceived(ctx, method, target, version): + + ctx.client = None + ctx.clientproto = version + + ctx.setHeader('Content-Length', 29) + ctx.write('shortContentLengthDisconnect') + + # Disconnect before the proxy sends the response headers + ctx.transport.loseConnection() + +server = tcp.Port(0, factory()) +server.startListening() + +print '# Listening on {0}:{1}'.format(*server.socket.getsockname()) + +class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + + print 'Bail out!' + reason.printTraceback() + + reactor.stop() + + class protocol(http.HTTPClient): + def connectionLost(ctx, reason): + try: + reactor.stop() + + except error.ReactorNotRunning: + pass + + else: + print 'ok 2 - The client connection closed' + + connectionMade = lambda ctx: ctx.transport.write('GET {0}:{1} HTTP/1.1\r\n\r\n'.format(*server.socket.getsockname())) + + def handleHeader(ctx, k, v): + if k.lower() == 'content-length': + if v != '29': + print 'not', + + # Who cares what happens now? + reactor.stop() + + print 'ok 1 - Content-Length header' + + # Avoid calling undefined handleResponse() at the end of the content (if + # the proxy sent the wrong Content-Length header). (Override + # connectionLost() when the proxy closes the client connection or stop the + # reactor.) + def handleResponseEnd(ctx): + pass + +tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect() + +reactor.run() diff --git a/plugins/experimental/metalink/test/zero b/plugins/experimental/metalink/test/zero new file mode 100755 index 00000000000..9b7b6fc3947 --- /dev/null +++ b/plugins/experimental/metalink/test/zero @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +print '''1..1 zero +# The proxy doesn't crash if the Content-Length is zero''' + +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + +def callback(): + print 'not ok 1 - Why didn\'t the test finish yet?' + + reactor.stop() + +reactor.callLater(1, callback) + +class factory(http.HTTPFactory): + class protocol(http.HTTPChannel): + class requestFactory(http.Request): + def requestReceived(ctx, method, target, version): + + ctx.client = None + ctx.clientproto = version + + ctx.setHeader('Content-Length', 0) + ctx.finish() + +server = tcp.Port(0, factory()) +server.startListening() + +print '# Listening on {0}:{1}'.format(*server.socket.getsockname()) + +class factory(protocol.ClientFactory): + def clientConnectionFailed(ctx, connector, reason): + + print 'Bail out!' + reason.printTraceback() + + reactor.stop() + + class protocol(http.HTTPClient): + def connectionLost(ctx, reason): + try: + reactor.stop() + + except error.ReactorNotRunning: + pass + + else: + print 'not ok 1 - Did the proxy crash? (The client connection closed.)' + + connectionMade = lambda ctx: ctx.transport.write('GET {0}:{1} HTTP/1.1\r\n\r\n'.format(*server.socket.getsockname())) + + def handleEndHeaders(ctx): + try: + reactor.stop() + + except error.ReactorNotRunning: + pass + + else: + print 'not ok 1 - No Content-Length header' + + def handleHeader(ctx, k, v): + if k.lower() == 'content-length': + if v != '0': + print 'not', + + print 'ok 1 - Content-Length header' + + reactor.stop() + +tcp.Connector('localhost', 8080, factory(), 30, None, reactor).connect() + +reactor.run() From 888b955720ad977a479d7a2549f3cdc18bcc24c1 Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Thu, 20 Feb 2014 13:22:36 -0800 Subject: [PATCH 15/16] metalink: Add Makefile target to run all the tests --- plugins/experimental/metalink/Makefile.am | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/experimental/metalink/Makefile.am b/plugins/experimental/metalink/Makefile.am index 490c980024a..dcff420636a 100644 --- a/plugins/experimental/metalink/Makefile.am +++ b/plugins/experimental/metalink/Makefile.am @@ -19,3 +19,9 @@ include $(top_srcdir)/build/plugins.mk pkglib_LTLIBRARIES = metalink.la metalink_la_SOURCES = metalink.cc metalink_la_LDFLAGS = $(TS_PLUGIN_LDFLAGS) + +check: + for e in test/*; do $$e; done | sed ' #\ + s/^ok [0-9]\+/\x1b[1;32m\0\x1b[0m/ #\ + s/^not ok [0-9]\|Bail out!\+/\x1b[1;37;41m\0\x1b[0m/ #\ + s/#.*/\x1b[33m\0\x1b[0m/' From 2017cc7935c1cd12870e28e50cf35174dd87dec7 Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Thu, 20 Feb 2014 13:35:25 -0800 Subject: [PATCH 16/16] metalink: Declare variables outside switch statement, for consistency --- plugins/experimental/metalink/metalink.cc | 34 +++++++++++------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/plugins/experimental/metalink/metalink.cc b/plugins/experimental/metalink/metalink.cc index 65f00a7ac6c..ea253dd4686 100644 --- a/plugins/experimental/metalink/metalink.cc +++ b/plugins/experimental/metalink/metalink.cc @@ -572,6 +572,12 @@ digest_handler(TSCont contp, TSEvent event, void *edata) static int location_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) { + const char *value; + int length; + + /* ATS_BASE64_DECODE_DSTLEN() */ + char digest[33]; + SendData *data = (SendData *) TSContDataGet(contp); TSContDestroy(contp); @@ -582,28 +588,20 @@ location_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) /* No: Check if the "Digest: SHA-256=..." digest already exists in the cache */ case TS_EVENT_CACHE_OPEN_READ_FAILED: - { - const char *value; - int length; - /* ATS_BASE64_DECODE_DSTLEN() */ - char digest[33]; + value = TSMimeHdrFieldValueStringGet(data->resp_bufp, data->hdr_loc, data->digest_loc, data->idx, &length); + if (TSBase64Decode(value + 8, length - 8, (unsigned char *) digest, sizeof(digest), NULL) != TS_SUCCESS + || TSCacheKeyDigestSet(data->key, digest, 32 /* SHA-256 */ ) != TS_SUCCESS) { + break; + } - value = TSMimeHdrFieldValueStringGet(data->resp_bufp, data->hdr_loc, data->digest_loc, data->idx, &length); - if (TSBase64Decode(value + 8, length - 8, (unsigned char *) digest, sizeof(digest), NULL) != TS_SUCCESS - || TSCacheKeyDigestSet(data->key, digest, 32 /* SHA-256 */ ) != TS_SUCCESS) { - break; - } + contp = TSContCreate(digest_handler, NULL); + TSContDataSet(contp, data); - contp = TSContCreate(digest_handler, NULL); - TSContDataSet(contp, data); - - TSCacheRead(contp, data->key); - TSHandleMLocRelease(data->resp_bufp, data->hdr_loc, data->digest_loc); + TSCacheRead(contp, data->key); + TSHandleMLocRelease(data->resp_bufp, data->hdr_loc, data->digest_loc); - return 0; - } - break; + return 0; default: TSAssert(!"Unexpected event");