From b3bf9163d3191b7f9085ef629da67581c2f97bca Mon Sep 17 00:00:00 2001 From: Qing Date: Mon, 3 Dec 2018 21:38:42 -0800 Subject: [PATCH 1/4] Reduce post processing time --- .../scala/org/apache/mxnet/infer/ObjectDetector.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/ObjectDetector.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/ObjectDetector.scala index a9b21f8c1dcd..dc4149188522 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/ObjectDetector.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/ObjectDetector.scala @@ -94,13 +94,13 @@ class ObjectDetector(modelPathPrefix: String, def objectDetectWithNDArray(input: IndexedSeq[NDArray], topK: Option[Int]) : IndexedSeq[IndexedSeq[(String, Array[Float])]] = { - val predictResult = predictor.predictWithNDArray(input)(0) + val predictResult = predictor.predictWithNDArray(input)(0).asInContext(Context.cpu()) var batchResult = ListBuffer[IndexedSeq[(String, Array[Float])]]() - for (i <- 0 until predictResult.shape(0)) { + (0 until predictResult.shape(0)).toVector.par.foreach( i => { val r = predictResult.at(i) batchResult += sortAndReformat(r, topK) handler.execute(r.dispose()) - } + }) handler.execute(predictResult.dispose()) batchResult.toIndexedSeq } @@ -113,7 +113,7 @@ class ObjectDetector(modelPathPrefix: String, // iterating over the all the predictions val length = predictResultND.shape(0) - for (i <- 0 until length) { + (0 until length).toVector.par.foreach( i => { val r = predictResultND.at(i) val tempArr = r.toArray if (tempArr(0) != -1.0) { @@ -123,7 +123,7 @@ class ObjectDetector(modelPathPrefix: String, // Ignore the minus 1 part } handler.execute(r.dispose()) - } + }) var result = IndexedSeq[(String, Array[Float])]() if (topK.isDefined) { var sortedIndices = accuracy.zipWithIndex.sortBy(-_._1).map(_._2) From c41057c6be07e9b8fc55349777d22967adf9b7b8 Mon Sep 17 00:00:00 2001 From: Qing Date: Tue, 4 Dec 2018 11:15:12 -0800 Subject: [PATCH 2/4] fix ssd --- .../src/main/scala/org/apache/mxnet/infer/Classifier.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/Classifier.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/Classifier.scala index adeb33d34a95..9c5a1d734699 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/Classifier.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/Classifier.scala @@ -110,16 +110,17 @@ class Classifier(modelPathPrefix: String, : IndexedSeq[IndexedSeq[(String, Float)]] = { // considering only the first output - val predictResultND: NDArray = predictor.predictWithNDArray(input)(0) + val predictResultND: NDArray = + predictor.predictWithNDArray(input)(0).asInContext(Context.cpu()) val predictResult: ListBuffer[Array[Float]] = ListBuffer[Array[Float]]() // iterating over the individual items(batch size is in axis 0) - for (i <- 0 until predictResultND.shape(0)) { + (0 until predictResultND.shape(0)).toVector.par.foreach( i => { val r = predictResultND.at(i) predictResult += r.toArray r.dispose() - } + }) var result: ListBuffer[IndexedSeq[(String, Float)]] = ListBuffer.empty[IndexedSeq[(String, Float)]] From 5ef1ee0732fb99c675df1b4693e8f8d3aca67594 Mon Sep 17 00:00:00 2001 From: Qing Date: Tue, 4 Dec 2018 15:48:03 -0800 Subject: [PATCH 3/4] fix the CI --- .../org/apache/mxnet/infer/Classifier.scala | 8 ++++-- .../apache/mxnet/infer/ObjectDetector.scala | 25 +++++++++---------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/Classifier.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/Classifier.scala index 9c5a1d734699..8351cf938695 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/Classifier.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/Classifier.scala @@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory import scala.io import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParArray trait ClassifierBase { @@ -113,15 +114,18 @@ class Classifier(modelPathPrefix: String, val predictResultND: NDArray = predictor.predictWithNDArray(input)(0).asInContext(Context.cpu()) - val predictResult: ListBuffer[Array[Float]] = ListBuffer[Array[Float]]() + val predictResultPar: ParArray[Array[Float]] = + new ParArray[Array[Float]](predictResultND.shape(0)) // iterating over the individual items(batch size is in axis 0) (0 until predictResultND.shape(0)).toVector.par.foreach( i => { val r = predictResultND.at(i) - predictResult += r.toArray + predictResultPar(i) = r.toArray r.dispose() }) + val predictResult = predictResultPar.toArray + var result: ListBuffer[IndexedSeq[(String, Float)]] = ListBuffer.empty[IndexedSeq[(String, Float)]] diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/ObjectDetector.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/ObjectDetector.scala index dc4149188522..244f0286f2fd 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/ObjectDetector.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/ObjectDetector.scala @@ -19,6 +19,8 @@ package org.apache.mxnet.infer // scalastyle:off import java.awt.image.BufferedImage + +import scala.collection.parallel.mutable.ParArray // scalastyle:on import org.apache.mxnet.NDArray import org.apache.mxnet.DataDesc @@ -95,10 +97,10 @@ class ObjectDetector(modelPathPrefix: String, : IndexedSeq[IndexedSeq[(String, Array[Float])]] = { val predictResult = predictor.predictWithNDArray(input)(0).asInContext(Context.cpu()) - var batchResult = ListBuffer[IndexedSeq[(String, Array[Float])]]() - (0 until predictResult.shape(0)).toVector.par.foreach( i => { + var batchResult = new ParArray[IndexedSeq[(String, Array[Float])]](predictResult.shape(0)) + (0 until predictResult.shape(0)).toArray.par.foreach( i => { val r = predictResult.at(i) - batchResult += sortAndReformat(r, topK) + batchResult(i) = sortAndReformat(r, topK) handler.execute(r.dispose()) }) handler.execute(predictResult.dispose()) @@ -107,26 +109,24 @@ class ObjectDetector(modelPathPrefix: String, private[infer] def sortAndReformat(predictResultND: NDArray, topK: Option[Int]) : IndexedSeq[(String, Array[Float])] = { - val predictResult: ListBuffer[Array[Float]] = ListBuffer[Array[Float]]() - val accuracy: ListBuffer[Float] = ListBuffer[Float]() - // iterating over the all the predictions val length = predictResultND.shape(0) - (0 until length).toVector.par.foreach( i => { + val predictResult = (0 until length).toArray.par.flatMap( i => { val r = predictResultND.at(i) val tempArr = r.toArray - if (tempArr(0) != -1.0) { - predictResult += tempArr - accuracy += tempArr(1) + val res = if (tempArr(0) != -1.0) { + Array[Array[Float]](tempArr) } else { // Ignore the minus 1 part + Array[Array[Float]]() } handler.execute(r.dispose()) - }) + res + }).toArray var result = IndexedSeq[(String, Array[Float])]() if (topK.isDefined) { - var sortedIndices = accuracy.zipWithIndex.sortBy(-_._1).map(_._2) + var sortedIndices = predictResult.zipWithIndex.sortBy(-_._1(1)).map(_._2) sortedIndices = sortedIndices.take(topK.get) // takeRight(5) would provide the output as Array[Accuracy, Xmin, Ymin, Xmax, Ymax result = sortedIndices.map(idx @@ -136,7 +136,6 @@ class ObjectDetector(modelPathPrefix: String, result = predictResult.map(ele => (synset(ele(0).toInt), ele.takeRight(5))).toIndexedSeq } - result } From a6673b5f735c40a8216c1368a716204a69ed332c Mon Sep 17 00:00:00 2001 From: Qing Date: Tue, 4 Dec 2018 22:58:53 -0800 Subject: [PATCH 4/4] add comments --- .../src/main/scala/org/apache/mxnet/infer/Classifier.scala | 3 ++- .../src/main/scala/org/apache/mxnet/infer/ObjectDetector.scala | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/Classifier.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/Classifier.scala index 8351cf938695..cf55bc10d97e 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/Classifier.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/Classifier.scala @@ -111,9 +111,10 @@ class Classifier(modelPathPrefix: String, : IndexedSeq[IndexedSeq[(String, Float)]] = { // considering only the first output + // Copy NDArray to CPU to avoid frequent GPU to CPU copying val predictResultND: NDArray = predictor.predictWithNDArray(input)(0).asInContext(Context.cpu()) - + // Parallel Execution with ParArray for better performance val predictResultPar: ParArray[Array[Float]] = new ParArray[Array[Float]](predictResultND.shape(0)) diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/ObjectDetector.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/ObjectDetector.scala index 244f0286f2fd..78b237a4a9c6 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/ObjectDetector.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/ObjectDetector.scala @@ -96,7 +96,9 @@ class ObjectDetector(modelPathPrefix: String, def objectDetectWithNDArray(input: IndexedSeq[NDArray], topK: Option[Int]) : IndexedSeq[IndexedSeq[(String, Array[Float])]] = { + // Copy NDArray to CPU to avoid frequent GPU to CPU copying val predictResult = predictor.predictWithNDArray(input)(0).asInContext(Context.cpu()) + // Parallel Execution with ParArray for better performance var batchResult = new ParArray[IndexedSeq[(String, Array[Float])]](predictResult.shape(0)) (0 until predictResult.shape(0)).toArray.par.foreach( i => { val r = predictResult.at(i)