From 471be8754cf6f77feb0549c72869422d12089e95 Mon Sep 17 00:00:00 2001 From: Sean Owen Date: Tue, 3 Mar 2015 13:22:59 +0000 Subject: [PATCH 1/2] Explicit check to see if JAR_CMD is available --- bin/compute-classpath.sh | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index f4f6b7b909490..ec84fc66d088e 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -93,14 +93,17 @@ if [ "$num_jars" -gt "1" ]; then exit 1 fi -# Verify that versions of java used to build the jars and run Spark are compatible -jar_error_check=$("$JAR_CMD" -tf "$ASSEMBLY_JAR" nonexistent/class/path 2>&1) -if [[ "$jar_error_check" =~ "invalid CEN header" ]]; then - echo "Loading Spark jar with '$JAR_CMD' failed. " 1>&2 - echo "This is likely because Spark was compiled with Java 7 and run " 1>&2 - echo "with Java 6. (see SPARK-1703). Please use Java 7 to run Spark " 1>&2 - echo "or build Spark with Java 6." 1>&2 - exit 1 +# Only able to make this check if 'jar' command is available +if [ $(command -v "$JAR_CMD") ] ; then + # Verify that versions of java used to build the jars and run Spark are compatible + jar_error_check=$("$JAR_CMD" -tf "$ASSEMBLY_JAR" nonexistent/class/path 2>&1) + if [[ "$jar_error_check" =~ "invalid CEN header" ]]; then + echo "Loading Spark jar with '$JAR_CMD' failed. " 1>&2 + echo "This is likely because Spark was compiled with Java 7 and run " 1>&2 + echo "with Java 6. (see SPARK-1703). Please use Java 7 to run Spark " 1>&2 + echo "or build Spark with Java 6." 1>&2 + exit 1 + fi fi CLASSPATH="$CLASSPATH:$ASSEMBLY_JAR" @@ -121,10 +124,15 @@ datanucleus_jars="$(find "$datanucleus_dir" 2>/dev/null | grep "datanucleus-.*\\ datanucleus_jars="$(echo "$datanucleus_jars" | tr "\n" : | sed s/:$//g)" if [ -n "$datanucleus_jars" ]; then - hive_files=$("$JAR_CMD" -tf "$ASSEMBLY_JAR" org/apache/hadoop/hive/ql/exec 2>/dev/null) - if [ -n "$hive_files" ]; then - echo "Spark assembly has been built with Hive, including Datanucleus jars on classpath" 1>&2 - CLASSPATH="$CLASSPATH:$datanucleus_jars" + if [ $(command -v "$JAR_CMD") ] ; then + hive_files=$("$JAR_CMD" -tf "$ASSEMBLY_JAR" org/apache/hadoop/hive/ql/exec 2>/dev/null) + if [ -n "$hive_files" ]; then + echo "Spark assembly has been built with Hive, including Datanucleus jars on classpath" 1>&2 + CLASSPATH="$CLASSPATH:$datanucleus_jars" + fi + else + echo "No jar command available; unable to check if assembly was built with Hive." + echo "Please install a JDK instead of a JRE." fi fi From 18b53a01cdaf471580497c81629625173194b62d Mon Sep 17 00:00:00 2001 From: Sean Owen Date: Mon, 9 Mar 2015 14:29:04 +0000 Subject: [PATCH 2/2] Alternative: don't check for Hive classes with jar, and always include datanucleus if present --- bin/compute-classpath.sh | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index ec84fc66d088e..fffff2846ec04 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -124,16 +124,7 @@ datanucleus_jars="$(find "$datanucleus_dir" 2>/dev/null | grep "datanucleus-.*\\ datanucleus_jars="$(echo "$datanucleus_jars" | tr "\n" : | sed s/:$//g)" if [ -n "$datanucleus_jars" ]; then - if [ $(command -v "$JAR_CMD") ] ; then - hive_files=$("$JAR_CMD" -tf "$ASSEMBLY_JAR" org/apache/hadoop/hive/ql/exec 2>/dev/null) - if [ -n "$hive_files" ]; then - echo "Spark assembly has been built with Hive, including Datanucleus jars on classpath" 1>&2 - CLASSPATH="$CLASSPATH:$datanucleus_jars" - fi - else - echo "No jar command available; unable to check if assembly was built with Hive." - echo "Please install a JDK instead of a JRE." - fi + CLASSPATH="$CLASSPATH:$datanucleus_jars" fi # Add test classes if we're running from SBT or Maven with SPARK_TESTING set to 1