From 8783b1ff2d9f81b3646f5adf3a572564faeba270 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 20 Apr 2020 21:32:32 +0000 Subject: [PATCH 1/5] When calling Urllib3HttpConnection.perform_request(), decode the response with surrogatepass error handler. --- elasticsearch/connection/http_urllib3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch/connection/http_urllib3.py b/elasticsearch/connection/http_urllib3.py index a4c5a268d..d1a5c4c9f 100644 --- a/elasticsearch/connection/http_urllib3.py +++ b/elasticsearch/connection/http_urllib3.py @@ -229,7 +229,7 @@ def perform_request( method, url, body, retries=Retry(False), headers=request_headers, **kw ) duration = time.time() - start - raw_data = response.data.decode("utf-8") + raw_data = response.data.decode("utf-8", "surrogatepass") except Exception as e: self.log_request_fail( method, full_url, url, orig_body, time.time() - start, exception=e From 8abac514610d6374c2a3f2897f5c19d0d3fc093a Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 20 Apr 2020 21:58:51 +0000 Subject: [PATCH 2/5] Add a surrogatepass test for the urllib3 connection helper. --- test_elasticsearch/test_connection.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test_elasticsearch/test_connection.py b/test_elasticsearch/test_connection.py index 8e5a9cda6..4252d31bd 100644 --- a/test_elasticsearch/test_connection.py +++ b/test_elasticsearch/test_connection.py @@ -397,6 +397,11 @@ def test_uncompressed_body_logged(self, logger): self.assertEquals('> {"example": "body"}', req[0][0] % req[0][1:]) self.assertEquals("< {}", resp[0][0] % resp[0][1:]) + def test_surrogatepass_into_bytes(self): + buf = b'\xe4\xbd\xa0\xe5\xa5\xbd\xed\xa9\xaa' + con = self._get_mock_connection(response_body=buf) + status, headers, data = con.perform_request("GET", "/") + self.assertEqual("你好\uda6a", data) class TestRequestsConnection(TestCase): def _get_mock_connection( From 69b6149e38f7b3bcb763837d8c5d72040e308f69 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 20 Apr 2020 22:11:13 +0000 Subject: [PATCH 3/5] Fix lint issue...maybe? --- test_elasticsearch/test_connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_elasticsearch/test_connection.py b/test_elasticsearch/test_connection.py index 4252d31bd..e8544f990 100644 --- a/test_elasticsearch/test_connection.py +++ b/test_elasticsearch/test_connection.py @@ -398,7 +398,7 @@ def test_uncompressed_body_logged(self, logger): self.assertEquals("< {}", resp[0][0] % resp[0][1:]) def test_surrogatepass_into_bytes(self): - buf = b'\xe4\xbd\xa0\xe5\xa5\xbd\xed\xa9\xaa' + buf = b"\xe4\xbd\xa0\xe5\xa5\xbd\xed\xa9\xaa" con = self._get_mock_connection(response_body=buf) status, headers, data = con.perform_request("GET", "/") self.assertEqual("你好\uda6a", data) From 0725849788b453d90440ab325dbebfaf4bf53f90 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 20 Apr 2020 22:12:32 +0000 Subject: [PATCH 4/5] Ok really fix the lint issue now. --- test_elasticsearch/test_connection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_elasticsearch/test_connection.py b/test_elasticsearch/test_connection.py index e8544f990..be55af32b 100644 --- a/test_elasticsearch/test_connection.py +++ b/test_elasticsearch/test_connection.py @@ -403,6 +403,7 @@ def test_surrogatepass_into_bytes(self): status, headers, data = con.perform_request("GET", "/") self.assertEqual("你好\uda6a", data) + class TestRequestsConnection(TestCase): def _get_mock_connection( self, connection_params={}, status_code=200, response_body="{}" From 35efa363ff2cd4bd6b5380ede87969cea8a4fbf0 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Tue, 21 Apr 2020 19:18:10 +0000 Subject: [PATCH 5/5] Add utf8 surrogatepass to http_requests --- elasticsearch/connection/http_requests.py | 2 +- test_elasticsearch/test_connection.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/elasticsearch/connection/http_requests.py b/elasticsearch/connection/http_requests.py index 4da838517..5db73aadf 100644 --- a/elasticsearch/connection/http_requests.py +++ b/elasticsearch/connection/http_requests.py @@ -140,7 +140,7 @@ def perform_request( try: response = self.session.send(prepared_request, **send_kwargs) duration = time.time() - start - raw_data = response.text + raw_data = response.content.decode("utf-8", "surrogatepass") except Exception as e: self.log_request_fail( method, diff --git a/test_elasticsearch/test_connection.py b/test_elasticsearch/test_connection.py index be55af32b..96c30491e 100644 --- a/test_elasticsearch/test_connection.py +++ b/test_elasticsearch/test_connection.py @@ -406,7 +406,7 @@ def test_surrogatepass_into_bytes(self): class TestRequestsConnection(TestCase): def _get_mock_connection( - self, connection_params={}, status_code=200, response_body="{}" + self, connection_params={}, status_code=200, response_body=b"{}" ): con = RequestsHttpConnection(**connection_params) @@ -414,7 +414,7 @@ def _dummy_send(*args, **kwargs): dummy_response = Mock() dummy_response.headers = {} dummy_response.status_code = status_code - dummy_response.text = response_body + dummy_response.content = response_body dummy_response.request = args[0] dummy_response.cookies = {} _dummy_send.call_args = (args, kwargs) @@ -659,7 +659,9 @@ def test_head_with_404_doesnt_get_logged(self, logger): @patch("elasticsearch.connection.base.tracer") @patch("elasticsearch.connection.base.logger") def test_failed_request_logs_and_traces(self, logger, tracer): - con = self._get_mock_connection(response_body='{"answer": 42}', status_code=500) + con = self._get_mock_connection( + response_body=b'{"answer": 42}', status_code=500 + ) self.assertRaises( TransportError, con.perform_request, @@ -685,7 +687,7 @@ def test_failed_request_logs_and_traces(self, logger, tracer): @patch("elasticsearch.connection.base.tracer") @patch("elasticsearch.connection.base.logger") def test_success_logs_and_traces(self, logger, tracer): - con = self._get_mock_connection(response_body="""{"answer": "that's it!"}""") + con = self._get_mock_connection(response_body=b"""{"answer": "that's it!"}""") status, headers, data = con.perform_request( "GET", "/", @@ -783,3 +785,9 @@ def test_url_prefix(self, tracer): "curl -H 'Content-Type: application/json' -XGET 'http://localhost:9200/_search?pretty' -d '{\n \"answer\": 42\n}'", tracer.info.call_args[0][0] % tracer.info.call_args[0][1:], ) + + def test_surrogatepass_into_bytes(self): + buf = b"\xe4\xbd\xa0\xe5\xa5\xbd\xed\xa9\xaa" + con = self._get_mock_connection(response_body=buf) + status, headers, data = con.perform_request("GET", "/") + self.assertEqual("你好\uda6a", data)