Skip to content
1 change: 1 addition & 0 deletions ci/rat-regex.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
.*\.config$
.*\.yaml$
.*\.gold$
.*\.test_input$
^\.gitignore$
^\.gitmodules$
^\.perltidyrc$
Expand Down
45 changes: 23 additions & 22 deletions proxy/hdrs/MIME.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2349,8 +2349,6 @@ ParseResult
MIMEScanner::get(TextView &input, TextView &output, bool &output_shares_input, bool eof_p, ScanType scan_type)
{
ParseResult zret = PARSE_RESULT_CONT;
// Need this for handling dangling CR.
static const char RAW_CR{ParseRules::CHAR_CR};

auto text = input;
while (PARSE_RESULT_CONT == zret && !text.empty()) {
Expand All @@ -2368,7 +2366,7 @@ MIMEScanner::get(TextView &input, TextView &output, bool &output_shares_input, b
}
} else if (ParseRules::is_lf(*text)) {
++text;
zret = PARSE_RESULT_DONE; // Required by regression test.
zret = PARSE_RESULT_ERROR; // lone LF
} else {
// consume this character in the next state.
m_state = MIME_PARSE_INSIDE;
Expand All @@ -2380,21 +2378,28 @@ MIMEScanner::get(TextView &input, TextView &output, bool &output_shares_input, b
++text;
zret = PARSE_RESULT_DONE;
} else {
// This really should be an error (spec doesn't permit lone CR) but the regression tests
// require it.
this->append(TextView(&RAW_CR, 1)); // This is to fix a core dump of the icc 19.1 compiler when {&RAW_CR, 1} is used
m_state = MIME_PARSE_INSIDE;
zret = PARSE_RESULT_ERROR; // lone CR
}
break;
case MIME_PARSE_INSIDE: {
auto lf_off = text.find(ParseRules::CHAR_LF);
if (lf_off != TextView::npos) {
text.remove_prefix(lf_off + 1); // drop up to and including LF
if (LINE == scan_type) {
zret = PARSE_RESULT_OK;
m_state = MIME_PARSE_BEFORE;
} else {
m_state = MIME_PARSE_AFTER; // looking for line folding.
auto cr_off = text.find(ParseRules::CHAR_CR);
if (cr_off != TextView::npos) {
text.remove_prefix(cr_off + 1); // drop up to and including CR
// Is the next item a LF?
if (!text.empty()) {
if (text[0] == ParseRules::CHAR_LF) {
text.remove_prefix(1); // drop up to and including LF
if (LINE == scan_type) {
zret = PARSE_RESULT_OK;
m_state = MIME_PARSE_BEFORE;
} else {
m_state = MIME_PARSE_AFTER; // looking for line folding.
}
} else { // Next char is not LF, you lose
zret = PARSE_RESULT_ERROR;
}
} else { // No LF yet, adjust state to note
m_state = MIME_PARSE_FOUND_CR;
}
} else { // no EOL, consume all text without changing state.
text.remove_prefix(text.size());
Expand Down Expand Up @@ -2531,18 +2536,14 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char
return err;
}

//////////////////////////////////////////////////
// if got a LF or CR on its own, end the header //
//////////////////////////////////////////////////
///////////////////////////////////////////////////
// if got a CR and LF on its own, end the header //
///////////////////////////////////////////////////

if ((parsed.size() >= 2) && (parsed[0] == ParseRules::CHAR_CR) && (parsed[1] == ParseRules::CHAR_LF)) {
return PARSE_RESULT_DONE;
}

if ((parsed.size() >= 1) && (parsed[0] == ParseRules::CHAR_LF)) {
return PARSE_RESULT_DONE;
}

/////////////////////////////////////////////
// find pointers into the name:value field //
/////////////////////////////////////////////
Expand Down
14 changes: 8 additions & 6 deletions proxy/hdrs/unit_tests/test_Hdrs.cc
Original file line number Diff line number Diff line change
Expand Up @@ -45,24 +45,26 @@ TEST_CASE("HdrTestHttpParse", "[proxy][hdrtest]")
int expected_result;
int expected_bytes_consumed;
};
static const std::array<Test, 21> tests = {{
static const std::array<Test, 23> tests = {{
{"GET /index.html HTTP/1.0\r\n", PARSE_RESULT_DONE, 26},
{"GET /index.html HTTP/1.0\r\n\r\n***BODY****", PARSE_RESULT_DONE, 28},
{"GET /index.html HTTP/1.0\r\na\rb\n", PARSE_RESULT_ERROR, 28},
{"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r \n", PARSE_RESULT_ERROR, 45},
{"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n\r\n***BODY****", PARSE_RESULT_DONE, 48},
{"GET", PARSE_RESULT_ERROR, 3},
{"GET /index.html", PARSE_RESULT_ERROR, 15},
{"GET /index.html\r\n", PARSE_RESULT_ERROR, 17},
{"GET /index.html HTTP/1.0", PARSE_RESULT_ERROR, 24},
{"GET /index.html HTTP/1.0\r", PARSE_RESULT_ERROR, 25},
{"GET /index.html HTTP/1.0\n", PARSE_RESULT_DONE, 25},
{"GET /index.html HTTP/1.0\n\n", PARSE_RESULT_DONE, 26},
{"GET /index.html HTTP/1.0\n", PARSE_RESULT_ERROR, 25},
{"GET /index.html HTTP/1.0\n\n", PARSE_RESULT_ERROR, 26},
{"GET /index.html HTTP/1.0\r\n\r\n", PARSE_RESULT_DONE, 28},
{"GET /index.html HTTP/1.0\r\nUser-Agent: foobar", PARSE_RESULT_ERROR, 44},
{"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\n", PARSE_RESULT_DONE, 45},
{"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\n", PARSE_RESULT_ERROR, 45},
{"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n", PARSE_RESULT_DONE, 46},
{"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n\r\n", PARSE_RESULT_DONE, 48},
{"GET /index.html HTTP/1.0\nUser-Agent: foobar\n", PARSE_RESULT_DONE, 44},
{"GET /index.html HTTP/1.0\nUser-Agent: foobar\nBoo: foo\n", PARSE_RESULT_DONE, 53},
{"GET /index.html HTTP/1.0\nUser-Agent: foobar\n", PARSE_RESULT_ERROR, 44},
{"GET /index.html HTTP/1.0\nUser-Agent: foobar\nBoo: foo\n", PARSE_RESULT_ERROR, 53},
{"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n", PARSE_RESULT_DONE, 46},
{"GET /index.html HTTP/1.0\r\n", PARSE_RESULT_DONE, 26},
{"GET /index.html hTTP/1.0\r\n", PARSE_RESULT_ERROR, 26},
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GET HTTP://www.customplugin204.test/ HTTP/1.1

GET HTTP://www.customplugin204.test/ HTTP/1.1
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GET HTTP://www.customtemplate204.test/ HTTP/1.1

GET HTTP://www.customtemplate204.test/ HTTP/1.1
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GET HTTP://www.default204.test/ HTTP/1.1

GET HTTP://www.default204.test/ HTTP/1.1
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GET HTTP://www.default304.test/ HTTP/1.1

GET HTTP://www.default304.test/ HTTP/1.1
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
GET /get200 HTTP/1.1
Host: www.example.test

GET /get200 HTTP/1.1
Host: www.example.test
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
GET /get304 HTTP/1.1
Host: www.example.test
If-Modified-Since: Thu, 1 Jan 1970 00:00:00 GMT

GET /get304 HTTP/1.1
Host: www.example.test
If-Modified-Since: Thu, 1 Jan 1970 00:00:00 GMT
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
HEAD http://www.example.test/ HTTP/1.1
Host: www.example.test

HEAD http://www.example.test/ HTTP/1.1
Host: www.example.test
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
HEAD /head200 HTTP/1.1
Host: www.example.test

HEAD /head200 HTTP/1.1
Host: www.example.test
4 changes: 2 additions & 2 deletions tests/gold_tests/body_factory/http204_response.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@
defaultTr.Processes.Default.StartBefore(Test.Processes.ts)
defaultTr.StillRunningAfter = ts

defaultTr.Processes.Default.Command = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{DEFAULT_204_HOST}_get.txt"
defaultTr.Processes.Default.Command = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{DEFAULT_204_HOST}_get.test_input"
defaultTr.Processes.Default.TimeOut = 5 # seconds
defaultTr.Processes.Default.ReturnCode = 0
defaultTr.Processes.Default.Streams.stdout = "gold/http-204.gold"
Expand All @@ -86,7 +86,7 @@
customTemplateTr = Test.AddTestRun(f"Test domain {CUSTOM_TEMPLATE_204_HOST}")
customTemplateTr.StillRunningBefore = ts
customTemplateTr.StillRunningAfter = ts
customTemplateTr.Processes.Default.Command = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{CUSTOM_TEMPLATE_204_HOST}_get.txt"
customTemplateTr.Processes.Default.Command = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{CUSTOM_TEMPLATE_204_HOST}_get.test_input"
customTemplateTr.Processes.Default.TimeOut = 5 # seconds
customTemplateTr.Processes.Default.ReturnCode = 0
customTemplateTr.Processes.Default.Streams.stdout = "gold/http-204-custom.gold"
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
tr.Processes.Default.StartBefore(Test.Processes.ts)
tr.StillRunningAfter = ts

tr.Processes.Default.Command = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{CUSTOM_PLUGIN_204_HOST}_get.txt"
tr.Processes.Default.Command = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{CUSTOM_PLUGIN_204_HOST}_get.test_input"
tr.Processes.Default.TimeOut = 5 # seconds
tr.Processes.Default.ReturnCode = 0
tr.Processes.Default.Streams.stdout = "gold/http-204-custom-plugin.gold"
2 changes: 1 addition & 1 deletion tests/gold_tests/body_factory/http304_response.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
tr.Processes.Default.StartBefore(Test.Processes.ts)
tr.StillRunningAfter = ts

cmd_tpl = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{DEFAULT_304_HOST}_get.txt"
cmd_tpl = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{DEFAULT_304_HOST}_get.test_input"
tr.Processes.Default.Command = cmd_tpl
tr.Processes.Default.TimeOut = 5 # seconds
tr.Processes.Default.ReturnCode = 0
Expand Down
2 changes: 1 addition & 1 deletion tests/gold_tests/body_factory/http_head_no_origin.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
tr.Processes.Default.StartBefore(Test.Processes.ts)
tr.StillRunningAfter = ts

tr.Processes.Default.Command = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{HOST}_head.txt"
tr.Processes.Default.Command = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{HOST}_head.test_input"
tr.Processes.Default.TimeOut = 5 # seconds
tr.Processes.Default.ReturnCode = 0
tr.Processes.Default.Streams.stdout = "gold/http-head-no-origin.gold"
6 changes: 3 additions & 3 deletions tests/gold_tests/body_factory/http_with_origin.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@
trhead200.StillRunningAfter = ts
trhead200.StillRunningAfter = server

trhead200.Processes.Default.Command = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{HOST}_head_200.txt"
trhead200.Processes.Default.Command = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{HOST}_head_200.test_input"
trhead200.Processes.Default.TimeOut = 5 # seconds
trhead200.Processes.Default.ReturnCode = 0
trhead200.Processes.Default.Streams.stdout = "gold/http-head-200.gold"
Expand All @@ -86,7 +86,7 @@
trget200.StillRunningAfter = ts
trget200.StillRunningAfter = server

trget200.Processes.Default.Command = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{HOST}_get_200.txt"
trget200.Processes.Default.Command = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{HOST}_get_200.test_input"
trget200.Processes.Default.TimeOut = 5 # seconds
trget200.Processes.Default.ReturnCode = 0
trget200.Processes.Default.Streams.stdout = "gold/http-get-200.gold"
Expand All @@ -98,7 +98,7 @@
trget304.StillRunningAfter = ts
trget304.StillRunningAfter = server

cmd_tpl = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{HOST}_get_304.txt"
cmd_tpl = f"{sys.executable} tcp_client.py 127.0.0.1 {ts.Variables.port} data/{HOST}_get_304.test_input"
trget304.Processes.Default.Command = cmd_tpl
trget304.Processes.Default.TimeOut = 5 # seconds
trget304.Processes.Default.ReturnCode = 0
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
POST / HTTP/1.1
Host: www.http408.test
Content-Length: 100

arbitrary content
POST / HTTP/1.1
Host: www.http408.test
Content-Length: 100
arbitrary content
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GET http://www.passthrough.test/ HTTP/1.1

GET http://www.passthrough.test/ HTTP/1.1
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GET http://www.redirect0.test/ HTTP/1.1

GET http://www.redirect0.test/ HTTP/1.1
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GET http://www.redirect301.test/ HTTP/1.1

GET http://www.redirect301.test/ HTTP/1.1
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GET http://www.redirect302.test/ HTTP/1.1

GET http://www.redirect302.test/ HTTP/1.1
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GET http://www.redirect307.test/ HTTP/1.1

GET http://www.redirect307.test/ HTTP/1.1
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GET http://www.redirect308.test/ HTTP/1.1

GET http://www.redirect308.test/ HTTP/1.1
12 changes: 6 additions & 6 deletions tests/gold_tests/headers/domain-blacklist-30x.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@
redirect301tr.Processes.Default.StartBefore(Test.Processes.ts)
redirect301tr.StillRunningAfter = ts
redirect301tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\
format(ts.Variables.port, 'data/{0}_get.txt'.format(REDIRECT_301_HOST))
format(ts.Variables.port, 'data/{0}_get.test_input'.format(REDIRECT_301_HOST))
redirect301tr.Processes.Default.TimeOut = 5 # seconds
redirect301tr.Processes.Default.ReturnCode = 0
redirect301tr.Processes.Default.Streams.stdout = "redirect301_get.gold"
Expand All @@ -69,7 +69,7 @@
redirect302tr.StillRunningBefore = ts
redirect302tr.StillRunningAfter = ts
redirect302tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\
format(ts.Variables.port, 'data/{0}_get.txt'.format(REDIRECT_302_HOST))
format(ts.Variables.port, 'data/{0}_get.test_input'.format(REDIRECT_302_HOST))
redirect302tr.Processes.Default.TimeOut = 5 # seconds
redirect302tr.Processes.Default.ReturnCode = 0
redirect302tr.Processes.Default.Streams.stdout = "redirect302_get.gold"
Expand All @@ -79,7 +79,7 @@
redirect302tr.StillRunningBefore = ts
redirect307tr.StillRunningAfter = ts
redirect307tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\
format(ts.Variables.port, 'data/{0}_get.txt'.format(REDIRECT_307_HOST))
format(ts.Variables.port, 'data/{0}_get.test_input'.format(REDIRECT_307_HOST))
redirect307tr.Processes.Default.TimeOut = 5 # seconds
redirect307tr.Processes.Default.ReturnCode = 0
redirect307tr.Processes.Default.Streams.stdout = "redirect307_get.gold"
Expand All @@ -88,7 +88,7 @@
redirect308tr.StillRunningBefore = ts
redirect308tr.StillRunningAfter = ts
redirect308tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\
format(ts.Variables.port, 'data/{0}_get.txt'.format(REDIRECT_308_HOST))
format(ts.Variables.port, 'data/{0}_get.test_input'.format(REDIRECT_308_HOST))
redirect308tr.Processes.Default.TimeOut = 5 # seconds
redirect308tr.Processes.Default.ReturnCode = 0
redirect308tr.Processes.Default.Streams.stdout = "redirect308_get.gold"
Expand All @@ -97,7 +97,7 @@
redirect0tr.StillRunningBefore = ts
redirect0tr.StillRunningAfter = ts
redirect0tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\
format(ts.Variables.port, 'data/{0}_get.txt'.format(REDIRECT_0_HOST))
format(ts.Variables.port, 'data/{0}_get.test_input'.format(REDIRECT_0_HOST))
redirect0tr.Processes.Default.TimeOut = 5 # seconds
redirect0tr.Processes.Default.ReturnCode = 0
redirect0tr.Processes.Default.Streams.stdout = "redirect0_get.gold"
Expand All @@ -106,7 +106,7 @@
passthroughtr.StillRunningBefore = ts
passthroughtr.StillRunningAfter = ts
passthroughtr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\
format(ts.Variables.port, 'data/{0}_get.txt'.format(PASSTHRU_HOST))
format(ts.Variables.port, 'data/{0}_get.test_input'.format(PASSTHRU_HOST))
passthroughtr.Processes.Default.TimeOut = 5 # seconds
passthroughtr.Processes.Default.ReturnCode = 0
passthroughtr.Processes.Default.Streams.stdout = "passthrough_get.gold"
Expand Down
2 changes: 1 addition & 1 deletion tests/gold_tests/headers/http408.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
tr.Processes.Default.StartBefore(server)
tr.Processes.Default.StartBefore(Test.Processes.ts)
tr.Processes.Default.Command = 'python3 tcp_client.py 127.0.0.1 {0} {1} --delay-after-send {2}'\
.format(ts.Variables.port, 'data/{0}.txt'.format(HTTP_408_HOST), TIMEOUT + 2)
.format(ts.Variables.port, 'data/{0}.test_input'.format(HTTP_408_HOST), TIMEOUT + 2)
tr.Processes.Default.ReturnCode = 0
tr.Processes.Default.TimeOut = 10
tr.Processes.Default.Streams.stdout = "http408.gold"
16 changes: 8 additions & 8 deletions tests/gold_tests/pluginTest/combo_handler/combo_handler.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,21 +117,21 @@ def add_server_obj(content_type, path):

tr = Test.AddTestRun()
tr.Processes.Default.Command = tcp_client("127.0.0.1", ts.Variables.port,
"GET /admin/v1/combo?obj1&sub:obj2&obj3 HTTP/1.1\n" +
"Host: xyz\n" +
"Connection: close\n" +
"\n"
"GET /admin/v1/combo?obj1&sub:obj2&obj3 HTTP/1.1\r\n" +
"Host: xyz\r\n" +
"Connection: close\r\n" +
"\r\n"
)
tr.Processes.Default.ReturnCode = 0
f = tr.Disk.File("_output/1-tr-Default/stream.all.txt")
f.Content = "combo_handler_files/tr1.gold"

tr = Test.AddTestRun()
tr.Processes.Default.Command = tcp_client("127.0.0.1", ts.Variables.port,
"GET /admin/v1/combo?obj1&sub:obj2&obj4 HTTP/1.1\n" +
"Host: xyz\n" +
"Connection: close\n" +
"\n"
"GET /admin/v1/combo?obj1&sub:obj2&obj4 HTTP/1.1\r\n" +
"Host: xyz\r\n" +
"Connection: close\r\n" +
"\r\n"
)
tr.Processes.Default.ReturnCode = 0
f = tr.Disk.File("_output/2-tr-Default/stream.all.txt")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
GET /argh HTTP/1.1
Host: none
X-Debug: X-Cache-Info

GET /argh HTTP/1.1
Host: none
X-Debug: X-Cache-Info
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
GET /argh HTTP/1.1
Host: one
X-Debug: X-Cache-Info

GET /argh HTTP/1.1
Host: one
X-Debug: X-Cache-Info
Loading