From 24c2c361c859e9233c31ff5918186f64de2c3571 Mon Sep 17 00:00:00 2001 From: Qing Date: Mon, 23 Jul 2018 17:29:35 -0700 Subject: [PATCH 1/6] add changes to example --- .../multitask/ExampleMultiTask.scala | 11 ++----- .../mxnetexamples/util/DownloadUtil.scala | 33 +++++++++++++++++++ .../CNNClassifierExampleSuite.scala | 22 ++++--------- .../customop/CustomOpExampleSuite.scala | 8 ++--- .../mxnetexamples/gan/GanExampleSuite.scala | 8 ++--- .../imclassification/MNISTExampleSuite.scala | 8 ++--- .../ImageClassifierExampleSuite.scala | 31 +++++------------ .../ObjectDetectorExampleSuite.scala | 30 +++++------------ 8 files changed, 70 insertions(+), 81 deletions(-) create mode 100644 scala-package/examples/src/main/scala/org/apache/mxnetexamples/util/DownloadUtil.scala diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/multitask/ExampleMultiTask.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/multitask/ExampleMultiTask.scala index 1270af3c45b4..d1d4cf5e0ce5 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/multitask/ExampleMultiTask.scala +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/multitask/ExampleMultiTask.scala @@ -24,9 +24,7 @@ import org.kohsuke.args4j.{CmdLineParser, Option} import org.slf4j.LoggerFactory import scala.collection.JavaConverters._ - import org.apache.commons.io.FileUtils - import org.apache.mxnet.Symbol import org.apache.mxnet.DataIter import org.apache.mxnet.DataBatch @@ -37,13 +35,13 @@ import org.apache.mxnet.Context import org.apache.mxnet.Xavier import org.apache.mxnet.optimizer.RMSProp import org.apache.mxnet.Executor +import org.apache.mxnetexamples.util.DownloadUtil import scala.collection.immutable.ListMap import scala.sys.process.Process /** * Example of multi-task - * @author Depeng Liang */ object ExampleMultiTask { private val logger = LoggerFactory.getLogger(classOf[ExampleMultiTask]) @@ -204,11 +202,8 @@ object ExampleMultiTask { val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci" val tempDirPath = System.getProperty("java.io.tmpdir") val modelDirPath = tempDirPath + File.separator + "multitask/" - val tmpFile = new File(tempDirPath + "/multitask/mnist.zip") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(baseUrl + "/mnist/mnist.zip"), - tmpFile) - } + DownloadUtil.downloadUrl(baseUrl + "/mnist/mnist.zip", + tempDirPath + "/multitask/mnist.zip") // TODO: Need to confirm with Windows Process("unzip " + tempDirPath + "/multitask/mnist.zip -d " diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/util/DownloadUtil.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/util/DownloadUtil.scala new file mode 100644 index 000000000000..9ee11f63ce23 --- /dev/null +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/util/DownloadUtil.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.mxnetexamples.util + +import java.io.File +import java.net.URL + +import org.apache.commons.io.FileUtils + +object DownloadUtil { + + def downloadUrl(url: String, filePath: String) : Unit = { + val tmpFile = new File(filePath) + if (!tmpFile.exists()) { + FileUtils.copyURLToFile(new URL(url), tmpFile) + } + } +} diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala index f7d1332792fb..3b22d7ce30b2 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala @@ -22,6 +22,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context +import org.apache.mxnetexamples.util.DownloadUtil import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory @@ -46,22 +47,13 @@ class CNNClassifierExampleSuite extends FunSuite with BeforeAndAfterAll { logger.info("Downloading CNN text...") val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala" - var tmpFile = new File(tempDirPath + "/CNN/rt-polarity.pos") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(baseUrl + "/scala-example-ci/CNN/rt-polarity.pos"), - tmpFile) - } - tmpFile = new File(tempDirPath + "/CNN/rt-polarity.neg") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(baseUrl + "/scala-example-ci/CNN/rt-polarity.neg"), - tmpFile) - } + DownloadUtil.downloadUrl(baseUrl + "/scala-example-ci/CNN/rt-polarity.pos", + tempDirPath + "/CNN/rt-polarity.pos") + DownloadUtil.downloadUrl(baseUrl + "/scala-example-ci/CNN/rt-polarity.neg", + tempDirPath + "/CNN/rt-polarity.neg") logger.info("Downloading pretrianed Word2Vec Model, may take a while") - tmpFile = new File(tempDirPath + "/CNN/" + w2vModelName) - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(baseUrl + "/scala-example-ci/CNN/" + w2vModelName), - tmpFile) - } + DownloadUtil.downloadUrl(baseUrl + "/scala-example-ci/CNN/" + w2vModelName, + tempDirPath + "/CNN/" + w2vModelName) val modelDirPath = tempDirPath + File.separator + "CNN" diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala index 4ba0e1bb87cb..aa5d3dd21a2f 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala @@ -21,6 +21,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context +import org.apache.mxnetexamples.util.DownloadUtil import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory @@ -64,11 +65,8 @@ class CustomOpExampleSuite extends FunSuite with BeforeAndAfterAll { val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci" val tempDirPath = System.getProperty("java.io.tmpdir") val modelDirPath = tempDirPath + File.separator + "mnist/" - val tmpFile = new File(tempDirPath + "/mnist/mnist.zip") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(baseUrl + "/mnist/mnist.zip"), - tmpFile) - } + DownloadUtil.downloadUrl(baseUrl + "/mnist/mnist.zip", + tempDirPath + "/mnist/mnist.zip") // TODO: Need to confirm with Windows Process("unzip " + tempDirPath + "/mnist/mnist.zip -d " + tempDirPath + "/mnist/") ! diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/gan/GanExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/gan/GanExampleSuite.scala index 12459fb1cc19..d1756f2b9a9a 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/gan/GanExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/gan/GanExampleSuite.scala @@ -22,6 +22,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context +import org.apache.mxnetexamples.util.DownloadUtil import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory @@ -38,11 +39,8 @@ class GanExampleSuite extends FunSuite with BeforeAndAfterAll{ val tempDirPath = System.getProperty("java.io.tmpdir") val modelDirPath = tempDirPath + File.separator + "mnist/" logger.info("tempDirPath: %s".format(tempDirPath)) - val tmpFile = new File(tempDirPath + "/mnist/mnist.zip") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(baseUrl + "/mnist/mnist.zip"), - tmpFile) - } + DownloadUtil.downloadUrl(baseUrl + "/mnist/mnist.zip", + tempDirPath + "/mnist/mnist.zip") // TODO: Need to confirm with Windows Process("unzip " + tempDirPath + "/mnist/mnist.zip -d " + tempDirPath + "/mnist/") ! diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala index 3e91b5b0245d..41cf4cb2ea1b 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala @@ -22,6 +22,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context +import org.apache.mxnetexamples.util.DownloadUtil import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory @@ -41,11 +42,8 @@ class MNISTExampleSuite extends FunSuite with BeforeAndAfterAll { val tempDirPath = System.getProperty("java.io.tmpdir") val modelDirPath = tempDirPath + File.separator + "mnist/" logger.info("tempDirPath: %s".format(tempDirPath)) - val tmpFile = new File(tempDirPath + "/mnist/mnist.zip") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(baseUrl + "/mnist/mnist.zip"), - tmpFile) - } + DownloadUtil.downloadUrl(baseUrl + "/mnist/mnist.zip", + tempDirPath + "/mnist/mnist.zip") // TODO: Need to confirm with Windows Process("unzip " + tempDirPath + "/mnist/mnist.zip -d " + tempDirPath + "/mnist/") ! diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala index 2b5ac7f8a2ae..75cdfd7d3951 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala @@ -24,6 +24,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context +import org.apache.mxnetexamples.util.DownloadUtil import sys.process.Process @@ -42,28 +43,14 @@ class ImageClassifierExampleSuite extends FunSuite with BeforeAndAfterAll { val baseUrl = "https://s3.us-east-2.amazonaws.com/scala-infer-models" - var tmpFile = new File(tempDirPath + "/resnet18/resnet-18-symbol.json") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(baseUrl + "/resnet-18/resnet-18-symbol.json"), - tmpFile) - } - tmpFile = new File(tempDirPath + "/resnet18/resnet-18-0000.params") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(baseUrl + "/resnet-18/resnet-18-0000.params"), - tmpFile) - } - tmpFile = new File(tempDirPath + "/resnet18/synset.txt") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(baseUrl + "/resnet-18/synset.txt"), - tmpFile) - } - tmpFile = new File(tempDirPath + "/inputImages/resnet18/Pug-Cookie.jpg") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile( - new URL("https://s3.amazonaws.com/model-server/inputs/Pug-Cookie.jpg"), - tmpFile - ) - } + DownloadUtil.downloadUrl(baseUrl + "/resnet-18/resnet-18-symbol.json", + tempDirPath + "/resnet18/resnet-18-symbol.json") + DownloadUtil.downloadUrl(baseUrl + "/resnet-18/resnet-18-0000.params", + tempDirPath + "/resnet18/resnet-18-0000.params") + DownloadUtil.downloadUrl(baseUrl + "/resnet-18/synset.txt", + tempDirPath + "/resnet18/synset.txt") + DownloadUtil.downloadUrl("https://s3.amazonaws.com/model-server/inputs/Pug-Cookie.jpg", + tempDirPath + "/inputImages/resnet18/Pug-Cookie.jpg") val modelDirPath = tempDirPath + File.separator + "resnet18/" val inputImagePath = tempDirPath + File.separator + diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/objectdetector/ObjectDetectorExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/objectdetector/ObjectDetectorExampleSuite.scala index 85b98381a433..9c8cb508ffb4 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/objectdetector/ObjectDetectorExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/objectdetector/ObjectDetectorExampleSuite.scala @@ -22,6 +22,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context +import org.apache.mxnetexamples.util.DownloadUtil import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory @@ -39,27 +40,14 @@ class ObjectDetectorExampleSuite extends FunSuite with BeforeAndAfterAll { val modelBase = "https://s3.amazonaws.com/model-server/models/resnet50_ssd/" val imageBase = "https://s3.amazonaws.com/model-server/inputs/" - - var tmpFile = new File(tempDirPath + "/resnetssd/resnet50_ssd_model-symbol.json") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(modelBase + "resnet50_ssd_model-symbol.json"), - tmpFile) - } - tmpFile = new File(tempDirPath + "/resnetssd/resnet50_ssd_model-0000.params") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(modelBase + "resnet50_ssd_model-0000.params"), - tmpFile) - } - tmpFile = new File(tempDirPath + "/resnetssd/synset.txt") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(modelBase + "synset.txt"), - tmpFile) - } - tmpFile = new File(tempDirPath + "/inputImages/resnetssd/dog-ssd.jpg") - if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(imageBase + "dog-ssd.jpg"), - tmpFile) - } + DownloadUtil.downloadUrl(modelBase + "resnet50_ssd_model-symbol.json", + tempDirPath + "/resnetssd/resnet50_ssd_model-symbol.json") + DownloadUtil.downloadUrl(modelBase + "resnet50_ssd_model-0000.params", + tempDirPath + "/resnetssd/resnet50_ssd_model-0000.params") + DownloadUtil.downloadUrl(modelBase + "synset.txt", + tempDirPath + "/resnetssd/synset.txt") + DownloadUtil.downloadUrl(imageBase + "dog-ssd.jpg", + tempDirPath + "/inputImages/resnetssd/dog-ssd.jpg") val modelDirPath = tempDirPath + File.separator + "resnetssd/" val inputImagePath = tempDirPath + File.separator + From 01566774f29bbd7a443f91985d7369d0ff4c86d0 Mon Sep 17 00:00:00 2001 From: Qing Date: Tue, 24 Jul 2018 23:47:14 -0700 Subject: [PATCH 2/6] place the file to the util --- scala-package/core/pom.xml | 5 +++++ .../src/main/scala/org/apache/mxnet}/util/DownloadUtil.scala | 2 +- .../apache/mxnetexamples/multitask/ExampleMultiTask.scala | 2 +- .../cnntextclassification/CNNClassifierExampleSuite.scala | 2 +- .../apache/mxnetexamples/customop/CustomOpExampleSuite.scala | 2 +- .../scala/org/apache/mxnetexamples/gan/GanExampleSuite.scala | 2 +- .../mxnetexamples/imclassification/MNISTExampleSuite.scala | 2 +- .../infer/imageclassifier/ImageClassifierExampleSuite.scala | 2 +- .../infer/objectdetector/ObjectDetectorExampleSuite.scala | 2 +- 9 files changed, 13 insertions(+), 8 deletions(-) rename scala-package/{examples/src/main/scala/org/apache/mxnetexamples => core/src/main/scala/org/apache/mxnet}/util/DownloadUtil.scala (96%) diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 134e0a59da1f..5095da0e8609 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -104,5 +104,10 @@ 1.3.0-SNAPSHOT provided + + commons-io + commons-io + 2.1 + diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/util/DownloadUtil.scala b/scala-package/core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala similarity index 96% rename from scala-package/examples/src/main/scala/org/apache/mxnetexamples/util/DownloadUtil.scala rename to scala-package/core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala index 9ee11f63ce23..b8f12bfc486e 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/util/DownloadUtil.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.mxnetexamples.util +package org.apache.mxnet.util import java.io.File import java.net.URL diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/multitask/ExampleMultiTask.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/multitask/ExampleMultiTask.scala index d1d4cf5e0ce5..280f4a9c0d7e 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/multitask/ExampleMultiTask.scala +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/multitask/ExampleMultiTask.scala @@ -35,7 +35,7 @@ import org.apache.mxnet.Context import org.apache.mxnet.Xavier import org.apache.mxnet.optimizer.RMSProp import org.apache.mxnet.Executor -import org.apache.mxnetexamples.util.DownloadUtil +import org.apache.mxnet.util.DownloadUtil import scala.collection.immutable.ListMap import scala.sys.process.Process diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala index 3b22d7ce30b2..7c2eb26dffeb 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala @@ -22,7 +22,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context -import org.apache.mxnetexamples.util.DownloadUtil +import org.apache.mxnet.util.DownloadUtil import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala index aa5d3dd21a2f..2a2528088dee 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala @@ -21,7 +21,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context -import org.apache.mxnetexamples.util.DownloadUtil +import org.apache.mxnet.util.DownloadUtil import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/gan/GanExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/gan/GanExampleSuite.scala index d1756f2b9a9a..a35009ebcdb4 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/gan/GanExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/gan/GanExampleSuite.scala @@ -22,7 +22,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context -import org.apache.mxnetexamples.util.DownloadUtil +import org.apache.mxnet.util.DownloadUtil import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala index 41cf4cb2ea1b..f411a0e10503 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala @@ -22,7 +22,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context -import org.apache.mxnetexamples.util.DownloadUtil +import org.apache.mxnet.util.DownloadUtil import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala index 75cdfd7d3951..b515424a88d4 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala @@ -24,7 +24,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context -import org.apache.mxnetexamples.util.DownloadUtil +import org.apache.mxnet.util.DownloadUtil import sys.process.Process diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/objectdetector/ObjectDetectorExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/objectdetector/ObjectDetectorExampleSuite.scala index 9c8cb508ffb4..8fc1ebfd413c 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/objectdetector/ObjectDetectorExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/objectdetector/ObjectDetectorExampleSuite.scala @@ -22,7 +22,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context -import org.apache.mxnetexamples.util.DownloadUtil +import org.apache.mxnet.util.DownloadUtil import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory From c75cd41dd4361e89bc6d2cfa237cef80c3561437 Mon Sep 17 00:00:00 2001 From: Qing Date: Wed, 25 Jul 2018 17:21:32 -0700 Subject: [PATCH 3/6] add retry scheme --- .../scala/org/apache/mxnet/util/DownloadUtil.scala | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala b/scala-package/core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala index b8f12bfc486e..662912d014ac 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala @@ -24,10 +24,20 @@ import org.apache.commons.io.FileUtils object DownloadUtil { - def downloadUrl(url: String, filePath: String) : Unit = { + def downloadUrl(url: String, filePath: String, maxRetry: Option[Int] = None) : Unit = { val tmpFile = new File(filePath) + var retry = maxRetry.getOrElse(3) + var success = false if (!tmpFile.exists()) { - FileUtils.copyURLToFile(new URL(url), tmpFile) + while (retry > 0 || success) { + try { + FileUtils.copyURLToFile(new URL(url), tmpFile) + success = true + } catch { + case e: Exception => retry -= 1 + } + } } + if (!success) throw new Exception(s"$url Download failed!") } } From b581bbf213201882dfe66f5c4e91e5de8363d5ec Mon Sep 17 00:00:00 2001 From: Qing Date: Wed, 25 Jul 2018 23:25:59 -0700 Subject: [PATCH 4/6] fix the retry logic --- .../src/main/scala/org/apache/mxnet/util/DownloadUtil.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala b/scala-package/core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala index 662912d014ac..f073153db2e1 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala @@ -29,7 +29,7 @@ object DownloadUtil { var retry = maxRetry.getOrElse(3) var success = false if (!tmpFile.exists()) { - while (retry > 0 || success) { + while (retry > 0 && !success) { try { FileUtils.copyURLToFile(new URL(url), tmpFile) success = true @@ -37,6 +37,8 @@ object DownloadUtil { case e: Exception => retry -= 1 } } + } else { + success = true } if (!success) throw new Exception(s"$url Download failed!") } From 5d5ad8424c870b265edc447b50f50d7829dbdfdd Mon Sep 17 00:00:00 2001 From: Qing Date: Fri, 27 Jul 2018 11:48:00 -0700 Subject: [PATCH 5/6] change the DownloadUtil to Util --- .../main/scala/org/apache/mxnetexamples/Util.scala} | 4 ++-- .../mxnetexamples/multitask/ExampleMultiTask.scala | 4 ++-- .../CNNClassifierExampleSuite.scala | 8 ++++---- .../mxnetexamples/customop/CustomOpExampleSuite.scala | 4 ++-- .../org/apache/mxnetexamples/gan/GanExampleSuite.scala | 4 ++-- .../imclassification/MNISTExampleSuite.scala | 4 ++-- .../imageclassifier/ImageClassifierExampleSuite.scala | 10 +++++----- .../objectdetector/ObjectDetectorExampleSuite.scala | 10 +++++----- 8 files changed, 24 insertions(+), 24 deletions(-) rename scala-package/{core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala => examples/src/main/scala/org/apache/mxnetexamples/Util.scala} (96%) diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/Util.scala similarity index 96% rename from scala-package/core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala rename to scala-package/examples/src/main/scala/org/apache/mxnetexamples/Util.scala index f073153db2e1..c1ff10c6c8a2 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/util/DownloadUtil.scala +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/Util.scala @@ -15,14 +15,14 @@ * limitations under the License. */ -package org.apache.mxnet.util +package org.apache.mxnetexamples import java.io.File import java.net.URL import org.apache.commons.io.FileUtils -object DownloadUtil { +object Util { def downloadUrl(url: String, filePath: String, maxRetry: Option[Int] = None) : Unit = { val tmpFile = new File(filePath) diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/multitask/ExampleMultiTask.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/multitask/ExampleMultiTask.scala index 280f4a9c0d7e..9df2bcc0566d 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/multitask/ExampleMultiTask.scala +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/multitask/ExampleMultiTask.scala @@ -35,7 +35,7 @@ import org.apache.mxnet.Context import org.apache.mxnet.Xavier import org.apache.mxnet.optimizer.RMSProp import org.apache.mxnet.Executor -import org.apache.mxnet.util.DownloadUtil +import org.apache.mxnetexamples.Util import scala.collection.immutable.ListMap import scala.sys.process.Process @@ -202,7 +202,7 @@ object ExampleMultiTask { val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci" val tempDirPath = System.getProperty("java.io.tmpdir") val modelDirPath = tempDirPath + File.separator + "multitask/" - DownloadUtil.downloadUrl(baseUrl + "/mnist/mnist.zip", + Util.downloadUrl(baseUrl + "/mnist/mnist.zip", tempDirPath + "/multitask/mnist.zip") // TODO: Need to confirm with Windows diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala index 7c2eb26dffeb..95c9823e3b28 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala @@ -22,7 +22,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context -import org.apache.mxnet.util.DownloadUtil +import org.apache.mxnetexamples.Util import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory @@ -47,12 +47,12 @@ class CNNClassifierExampleSuite extends FunSuite with BeforeAndAfterAll { logger.info("Downloading CNN text...") val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala" - DownloadUtil.downloadUrl(baseUrl + "/scala-example-ci/CNN/rt-polarity.pos", + Util.downloadUrl(baseUrl + "/scala-example-ci/CNN/rt-polarity.pos", tempDirPath + "/CNN/rt-polarity.pos") - DownloadUtil.downloadUrl(baseUrl + "/scala-example-ci/CNN/rt-polarity.neg", + Util.downloadUrl(baseUrl + "/scala-example-ci/CNN/rt-polarity.neg", tempDirPath + "/CNN/rt-polarity.neg") logger.info("Downloading pretrianed Word2Vec Model, may take a while") - DownloadUtil.downloadUrl(baseUrl + "/scala-example-ci/CNN/" + w2vModelName, + Util.downloadUrl(baseUrl + "/scala-example-ci/CNN/" + w2vModelName, tempDirPath + "/CNN/" + w2vModelName) val modelDirPath = tempDirPath + File.separator + "CNN" diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala index 2a2528088dee..6385e062a260 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala @@ -21,7 +21,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context -import org.apache.mxnet.util.DownloadUtil +import org.apache.mxnetexamples.Util import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory @@ -65,7 +65,7 @@ class CustomOpExampleSuite extends FunSuite with BeforeAndAfterAll { val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci" val tempDirPath = System.getProperty("java.io.tmpdir") val modelDirPath = tempDirPath + File.separator + "mnist/" - DownloadUtil.downloadUrl(baseUrl + "/mnist/mnist.zip", + Util.downloadUrl(baseUrl + "/mnist/mnist.zip", tempDirPath + "/mnist/mnist.zip") // TODO: Need to confirm with Windows Process("unzip " + tempDirPath + "/mnist/mnist.zip -d " diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/gan/GanExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/gan/GanExampleSuite.scala index a35009ebcdb4..8ab3a4b364a7 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/gan/GanExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/gan/GanExampleSuite.scala @@ -22,7 +22,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context -import org.apache.mxnet.util.DownloadUtil +import org.apache.mxnetexamples.Util import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory @@ -39,7 +39,7 @@ class GanExampleSuite extends FunSuite with BeforeAndAfterAll{ val tempDirPath = System.getProperty("java.io.tmpdir") val modelDirPath = tempDirPath + File.separator + "mnist/" logger.info("tempDirPath: %s".format(tempDirPath)) - DownloadUtil.downloadUrl(baseUrl + "/mnist/mnist.zip", + Util.downloadUrl(baseUrl + "/mnist/mnist.zip", tempDirPath + "/mnist/mnist.zip") // TODO: Need to confirm with Windows Process("unzip " + tempDirPath + "/mnist/mnist.zip -d " diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala index f411a0e10503..7b1d6ddc38b5 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala @@ -22,7 +22,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context -import org.apache.mxnet.util.DownloadUtil +import org.apache.mxnetexamples.Util import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory @@ -42,7 +42,7 @@ class MNISTExampleSuite extends FunSuite with BeforeAndAfterAll { val tempDirPath = System.getProperty("java.io.tmpdir") val modelDirPath = tempDirPath + File.separator + "mnist/" logger.info("tempDirPath: %s".format(tempDirPath)) - DownloadUtil.downloadUrl(baseUrl + "/mnist/mnist.zip", + Util.downloadUrl(baseUrl + "/mnist/mnist.zip", tempDirPath + "/mnist/mnist.zip") // TODO: Need to confirm with Windows Process("unzip " + tempDirPath + "/mnist/mnist.zip -d " diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala index b515424a88d4..f0bb07b4a398 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala @@ -24,7 +24,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context -import org.apache.mxnet.util.DownloadUtil +import org.apache.mxnetexamples.Util import sys.process.Process @@ -43,13 +43,13 @@ class ImageClassifierExampleSuite extends FunSuite with BeforeAndAfterAll { val baseUrl = "https://s3.us-east-2.amazonaws.com/scala-infer-models" - DownloadUtil.downloadUrl(baseUrl + "/resnet-18/resnet-18-symbol.json", + Util.downloadUrl(baseUrl + "/resnet-18/resnet-18-symbol.json", tempDirPath + "/resnet18/resnet-18-symbol.json") - DownloadUtil.downloadUrl(baseUrl + "/resnet-18/resnet-18-0000.params", + Util.downloadUrl(baseUrl + "/resnet-18/resnet-18-0000.params", tempDirPath + "/resnet18/resnet-18-0000.params") - DownloadUtil.downloadUrl(baseUrl + "/resnet-18/synset.txt", + Util.downloadUrl(baseUrl + "/resnet-18/synset.txt", tempDirPath + "/resnet18/synset.txt") - DownloadUtil.downloadUrl("https://s3.amazonaws.com/model-server/inputs/Pug-Cookie.jpg", + Util.downloadUrl("https://s3.amazonaws.com/model-server/inputs/Pug-Cookie.jpg", tempDirPath + "/inputImages/resnet18/Pug-Cookie.jpg") val modelDirPath = tempDirPath + File.separator + "resnet18/" diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/objectdetector/ObjectDetectorExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/objectdetector/ObjectDetectorExampleSuite.scala index 8fc1ebfd413c..31da38569281 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/objectdetector/ObjectDetectorExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/objectdetector/ObjectDetectorExampleSuite.scala @@ -22,7 +22,7 @@ import java.net.URL import org.apache.commons.io.FileUtils import org.apache.mxnet.Context -import org.apache.mxnet.util.DownloadUtil +import org.apache.mxnetexamples.Util import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.slf4j.LoggerFactory @@ -40,13 +40,13 @@ class ObjectDetectorExampleSuite extends FunSuite with BeforeAndAfterAll { val modelBase = "https://s3.amazonaws.com/model-server/models/resnet50_ssd/" val imageBase = "https://s3.amazonaws.com/model-server/inputs/" - DownloadUtil.downloadUrl(modelBase + "resnet50_ssd_model-symbol.json", + Util.downloadUrl(modelBase + "resnet50_ssd_model-symbol.json", tempDirPath + "/resnetssd/resnet50_ssd_model-symbol.json") - DownloadUtil.downloadUrl(modelBase + "resnet50_ssd_model-0000.params", + Util.downloadUrl(modelBase + "resnet50_ssd_model-0000.params", tempDirPath + "/resnetssd/resnet50_ssd_model-0000.params") - DownloadUtil.downloadUrl(modelBase + "synset.txt", + Util.downloadUrl(modelBase + "synset.txt", tempDirPath + "/resnetssd/synset.txt") - DownloadUtil.downloadUrl(imageBase + "dog-ssd.jpg", + Util.downloadUrl(imageBase + "dog-ssd.jpg", tempDirPath + "/inputImages/resnetssd/dog-ssd.jpg") val modelDirPath = tempDirPath + File.separator + "resnetssd/" From c4aea9922af4a3d55477a0da7e66624824176140 Mon Sep 17 00:00:00 2001 From: Qing Date: Fri, 27 Jul 2018 13:18:13 -0700 Subject: [PATCH 6/6] Trigger the CI