From 1d5204d53ea863e8aa13a26313419b880a4aa8ed Mon Sep 17 00:00:00 2001 From: Suneet Saldanha Date: Tue, 17 Dec 2019 17:06:53 -0800 Subject: [PATCH 1/3] Remove resolve-ip dependency for integration-tests --- integration-tests/README.md | 3 ++- .../tls/generate-expired-client-cert.sh | 4 +++- .../docker/tls/generate-good-client-cert.sh | 4 +++- ...generate-incorrect-hostname-client-cert.sh | 4 +++- ...nerate-invalid-intermediate-client-cert.sh | 6 ++++-- .../tls/generate-to-be-revoked-client-cert.sh | 4 +++- .../generate-untrusted-root-client-cert.sh | 4 +++- ...generate-valid-intermediate-client-cert.sh | 6 ++++-- .../docker/tls/set-docker-host-ip.sh | 19 +++++++++++++++++++ 9 files changed, 44 insertions(+), 10 deletions(-) create mode 100755 integration-tests/docker/tls/set-docker-host-ip.sh diff --git a/integration-tests/README.md b/integration-tests/README.md index 1f7b4de0e6ba..93ccfe5a20b3 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -36,7 +36,8 @@ Integration Testing Using Docker For running integration tests using docker there are 2 approaches. If your platform supports docker natively, you can simply set `DOCKER_IP` -environment variable to localhost and skip to [Running tests](#running-tests) section. +environment variable to localhost and skip to [Running tests](#running-tests) section. Ensure that you have +at least 4GiB of memory allocated to the docker engine (This can be set under Preferences > Advanced). ``` export DOCKER_IP=127.0.0.1 diff --git a/integration-tests/docker/tls/generate-expired-client-cert.sh b/integration-tests/docker/tls/generate-expired-client-cert.sh index 71fb8b7f633b..dd05847644a5 100755 --- a/integration-tests/docker/tls/generate-expired-client-cert.sh +++ b/integration-tests/docker/tls/generate-expired-client-cert.sh @@ -15,7 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -export DOCKER_HOST_IP=$(resolveip -s $HOSTNAME) +tls_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=set-docker-host-ip.sh +source "$tls_dir/set-docker-host-ip.sh" cat < expired_csr.conf [req] diff --git a/integration-tests/docker/tls/generate-good-client-cert.sh b/integration-tests/docker/tls/generate-good-client-cert.sh index e166d0908603..895e6c34bad8 100755 --- a/integration-tests/docker/tls/generate-good-client-cert.sh +++ b/integration-tests/docker/tls/generate-good-client-cert.sh @@ -15,7 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -export DOCKER_HOST_IP=$(resolveip -s $HOSTNAME) +tls_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=set-docker-host-ip.sh +source "$tls_dir/set-docker-host-ip.sh" cat < csr.conf [req] diff --git a/integration-tests/docker/tls/generate-incorrect-hostname-client-cert.sh b/integration-tests/docker/tls/generate-incorrect-hostname-client-cert.sh index b778aa2aa3ba..41a7a7d6bef5 100755 --- a/integration-tests/docker/tls/generate-incorrect-hostname-client-cert.sh +++ b/integration-tests/docker/tls/generate-incorrect-hostname-client-cert.sh @@ -15,7 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -export DOCKER_HOST_IP=$(resolveip -s $HOSTNAME) +tls_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=set-docker-host-ip.sh +source "$tls_dir/set-docker-host-ip.sh" # Generate a client cert with an incorrect hostname for testing cat < invalid_hostname_csr.conf diff --git a/integration-tests/docker/tls/generate-invalid-intermediate-client-cert.sh b/integration-tests/docker/tls/generate-invalid-intermediate-client-cert.sh index fc7771683250..4744e9f4a8f0 100755 --- a/integration-tests/docker/tls/generate-invalid-intermediate-client-cert.sh +++ b/integration-tests/docker/tls/generate-invalid-intermediate-client-cert.sh @@ -15,7 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -export DOCKER_HOST_IP=$(resolveip -s $HOSTNAME) +tls_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=set-docker-host-ip.sh +source "$tls_dir/set-docker-host-ip.sh" cat < invalid_ca_intermediate.conf [req] @@ -89,4 +91,4 @@ cat invalid_ca_intermediate.pem >> invalid_ca_client.pem # Create a Java keystore containing the generated certificate openssl pkcs12 -export -in invalid_ca_client.pem -inkey invalid_ca_client.key -out invalid_ca_client.p12 -name invalid_ca_client -CAfile invalid_ca_intermediate.pem -caname druid-it-root -password pass:druid123 -keytool -importkeystore -srckeystore invalid_ca_client.p12 -srcstoretype PKCS12 -destkeystore invalid_ca_client.jks -deststoretype JKS -srcstorepass druid123 -deststorepass druid123 \ No newline at end of file +keytool -importkeystore -srckeystore invalid_ca_client.p12 -srcstoretype PKCS12 -destkeystore invalid_ca_client.jks -deststoretype JKS -srcstorepass druid123 -deststorepass druid123 diff --git a/integration-tests/docker/tls/generate-to-be-revoked-client-cert.sh b/integration-tests/docker/tls/generate-to-be-revoked-client-cert.sh index effcad5e3099..e1d9c6687cca 100755 --- a/integration-tests/docker/tls/generate-to-be-revoked-client-cert.sh +++ b/integration-tests/docker/tls/generate-to-be-revoked-client-cert.sh @@ -15,7 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -export DOCKER_HOST_IP=$(resolveip -s $HOSTNAME) +tls_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=set-docker-host-ip.sh +source "$tls_dir/set-docker-host-ip.sh" # Generate a client cert that will be revoked cat < revoked_csr.conf diff --git a/integration-tests/docker/tls/generate-untrusted-root-client-cert.sh b/integration-tests/docker/tls/generate-untrusted-root-client-cert.sh index c133a5df1eab..b68c66f43be1 100755 --- a/integration-tests/docker/tls/generate-untrusted-root-client-cert.sh +++ b/integration-tests/docker/tls/generate-untrusted-root-client-cert.sh @@ -15,7 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -export DOCKER_HOST_IP=$(resolveip -s $HOSTNAME) +tls_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=set-docker-host-ip.sh +source "$tls_dir/set-docker-host-ip.sh" cat < csr_another_root.conf [req] diff --git a/integration-tests/docker/tls/generate-valid-intermediate-client-cert.sh b/integration-tests/docker/tls/generate-valid-intermediate-client-cert.sh index 914a7031100f..53e630db0213 100755 --- a/integration-tests/docker/tls/generate-valid-intermediate-client-cert.sh +++ b/integration-tests/docker/tls/generate-valid-intermediate-client-cert.sh @@ -15,7 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -export DOCKER_HOST_IP=$(resolveip -s $HOSTNAME) +tls_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=set-docker-host-ip.sh +source "$tls_dir/set-docker-host-ip.sh" cat < ca_intermediate.conf [req] @@ -89,4 +91,4 @@ cat ca_intermediate.pem >> intermediate_ca_client.pem # Create a Java keystore containing the generated certificate openssl pkcs12 -export -in intermediate_ca_client.pem -inkey intermediate_ca_client.key -out intermediate_ca_client.p12 -name intermediate_ca_client -CAfile ca_intermediate.pem -caname druid-it-root -password pass:druid123 -keytool -importkeystore -srckeystore intermediate_ca_client.p12 -srcstoretype PKCS12 -destkeystore intermediate_ca_client.jks -deststoretype JKS -srcstorepass druid123 -deststorepass druid123 \ No newline at end of file +keytool -importkeystore -srckeystore intermediate_ca_client.p12 -srcstoretype PKCS12 -destkeystore intermediate_ca_client.jks -deststoretype JKS -srcstorepass druid123 -deststorepass druid123 diff --git a/integration-tests/docker/tls/set-docker-host-ip.sh b/integration-tests/docker/tls/set-docker-host-ip.sh new file mode 100755 index 000000000000..85209432c8ad --- /dev/null +++ b/integration-tests/docker/tls/set-docker-host-ip.sh @@ -0,0 +1,19 @@ +#!/bin/bash -eu + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DOCKER_HOST_IP="$(dscacheutil -q host -a name "$(HOSTNAME)" | perl -nle '/ip_address: (.*)/ && print $1')" +export DOCKER_HOST_IP From f58826c5c784d0b6386354ac4a7033e9cfa3774f Mon Sep 17 00:00:00 2001 From: Suneet Saldanha Date: Wed, 18 Dec 2019 12:06:54 -0800 Subject: [PATCH 2/3] use host hostname and fallback to dscacheutil --- integration-tests/docker/tls/set-docker-host-ip.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integration-tests/docker/tls/set-docker-host-ip.sh b/integration-tests/docker/tls/set-docker-host-ip.sh index 85209432c8ad..818d67bddeab 100755 --- a/integration-tests/docker/tls/set-docker-host-ip.sh +++ b/integration-tests/docker/tls/set-docker-host-ip.sh @@ -15,5 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -DOCKER_HOST_IP="$(dscacheutil -q host -a name "$(HOSTNAME)" | perl -nle '/ip_address: (.*)/ && print $1')" +DOCKER_HOST_IP="$(host "$(hostname)" | perl -nle '/has address (.*)/ && print $1')" +if [ "$DOCKER_HOST_IP" -eq "" ]; then + DOCKER_HOST_IP="$(dscacheutil -q host -a name "$(HOSTNAME)" | perl -nle '/ip_address: (.*)/ && print $1')" +fi export DOCKER_HOST_IP From e28ccc7572a32c7f1a55c21794db942e46de3dfe Mon Sep 17 00:00:00 2001 From: Suneet Saldanha Date: Wed, 18 Dec 2019 13:45:49 -0800 Subject: [PATCH 3/3] better shell script comparisons --- integration-tests/docker/tls/set-docker-host-ip.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/integration-tests/docker/tls/set-docker-host-ip.sh b/integration-tests/docker/tls/set-docker-host-ip.sh index 818d67bddeab..38fe6ae6e395 100755 --- a/integration-tests/docker/tls/set-docker-host-ip.sh +++ b/integration-tests/docker/tls/set-docker-host-ip.sh @@ -16,7 +16,14 @@ # limitations under the License. DOCKER_HOST_IP="$(host "$(hostname)" | perl -nle '/has address (.*)/ && print $1')" -if [ "$DOCKER_HOST_IP" -eq "" ]; then +if [ -z "$DOCKER_HOST_IP" ]; then + # Mac specific way to get host ip DOCKER_HOST_IP="$(dscacheutil -q host -a name "$(HOSTNAME)" | perl -nle '/ip_address: (.*)/ && print $1')" fi + +if [ -z "$DOCKER_HOST_IP" ]; then + >&2 echo "Could not set docker host IP - integration tests can not run" + exit 1 +fi + export DOCKER_HOST_IP