From 1a1f14005afad01a9b8499dc59ecb29689a124ee Mon Sep 17 00:00:00 2001 From: Ayres Date: Mon, 15 Oct 2018 14:53:54 -0700 Subject: [PATCH 01/17] New Java inference API and SSD example --- .../objectdetector/run_ssd_java_example.sh | 35 ++++ .../infer/javapi/objectdetector/README.md | 116 ++++++++++++ .../objectdetector/SSDClassifierExample.java | 174 ++++++++++++++++++ .../infer/objectdetector/README.md | 4 +- .../mxnet/infer/javaapi/ObjectDetector.scala | 39 ++++ .../mxnet/infer/javaapi/Predictor.scala | 29 +++ 6 files changed, 395 insertions(+), 2 deletions(-) create mode 100755 scala-package/examples/scripts/infer/objectdetector/run_ssd_java_example.sh create mode 100644 scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/README.md create mode 100644 scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java create mode 100644 scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala create mode 100644 scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala diff --git a/scala-package/examples/scripts/infer/objectdetector/run_ssd_java_example.sh b/scala-package/examples/scripts/infer/objectdetector/run_ssd_java_example.sh new file mode 100755 index 000000000000..c7e58cebc513 --- /dev/null +++ b/scala-package/examples/scripts/infer/objectdetector/run_ssd_java_example.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +MXNET_ROOT=$(cd "$(dirname $0)/../../../../../"; pwd) +CLASS_PATH=$MXNET_ROOT/scala-package/assembly/osx-x86_64-cpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/*:$MXNET_ROOT/scala-package/infer/target/*:$MXNET_ROOT/scala-package/examples/src/main/scala/org/apache/mxnetexamples/api/java/infer/imageclassifier/* + +# model dir and prefix +MODEL_DIR=$1 +# input image +INPUT_IMG=$2 +# which input image dir +INPUT_DIR=$3 + +java -Xmx8G -cp $CLASS_PATH \ + org.apache.mxnetexamples.infer.javapi.objectdetector.SSDClassifierExample \ + --model-path-prefix $MODEL_DIR \ + --input-image $INPUT_IMG \ + --input-dir $INPUT_DIR diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/README.md b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/README.md new file mode 100644 index 000000000000..bf4a44a76d00 --- /dev/null +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/README.md @@ -0,0 +1,116 @@ +# Single Shot Multi Object Detection using Scala Inference API + +In this example, you will learn how to use Scala Inference API to run Inference on pre-trained Single Shot Multi Object Detection (SSD) MXNet model. + +The model is trained on the [Pascal VOC 2012 dataset](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/index.html). The network is a SSD model built on Resnet50 as base network to extract image features. The model is trained to detect the following entities (classes): ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']. For more details about the model, you can refer to the [MXNet SSD example](https://github.com/apache/incubator-mxnet/tree/master/example/ssd). + + +## Contents + +1. [Prerequisites](#prerequisites) +2. [Download artifacts](#download-artifacts) +3. [Setup datapath and parameters](#setup-datapath-and-parameters) +4. [Run the image inference example](#run-the-image-inference-example) +5. [Infer APIs](#infer-api-details) +6. [Next steps](#next-steps) + + +## Prerequisites + +1. MXNet +2. MXNet Scala Package +3. [IntelliJ IDE (or alternative IDE) project setup](http://mxnet.incubator.apache.org/tutorials/scala/mxnet_scala_on_intellij.html) with the MXNet Scala Package +4. wget + + +## Setup Guide + +### Download Artifacts +#### Step 1 +You can download the files using the script `get_ssd_data.sh`. It will download and place the model files in a `model` folder and the test image files in a `image` folder in the current directory. +From the `scala-package/examples/scripts/infer/imageclassifier/` folder run: + +```bash +./get_ssd_data.sh +``` + +**Note**: You may need to run `chmod +x get_resnet_data.sh` before running this script. + +Alternatively use the following links to download the Symbol and Params files via your browser: +- [resnet50_ssd_model-symbol.json](https://s3.amazonaws.com/model-server/models/resnet50_ssd/resnet50_ssd_model-symbol.json) +- [resnet50_ssd_model-0000.params](https://s3.amazonaws.com/model-server/models/resnet50_ssd/resnet50_ssd_model-0000.params) +- [synset.txt](https://github.com/awslabs/mxnet-model-server/blob/master/examples/ssd/synset.txt) + +In the pre-trained model, the `input_name` is `data` and shape is `(1, 3, 512, 512)`. +This shape translates to: a batch of `1` image, the image has color and uses `3` channels (RGB), and the image has the dimensions of `512` pixels in height by `512` pixels in width. + +`image/jpeg` is the expected input type, since this example's image pre-processor only supports the handling of binary JPEG images. + +The output shape is `(1, 6132, 6)`. As with the input, the `1` is the number of images. `6132` is the number of prediction results, and `6` is for the size of each prediction. Each prediction contains the following components: +- `Class` +- `Accuracy` +- `Xmin` +- `Ymin` +- `Xmax` +- `Ymax` + + +### Setup Datapath and Parameters +#### Step 2 +The code `Line 31: val baseDir = System.getProperty("user.dir")` in the example will automatically searches the work directory you have defined. Please put the files in your [work directory](https://stackoverflow.com/questions/16239130/java-user-dir-property-what-exactly-does-it-mean). + +Alternatively, if you would like to use your own path, please change line 31 into your own path +```scala +val baseDir = +``` + +The followings is the parameters defined for this example, you can find more information in the `class SSDClassifierExample`. + +| Argument | Comments | +| ----------------------------- | ---------------------------------------- | +| `model-path-prefix` | Folder path with prefix to the model (including json, params, and any synset file). | +| `input-image` | The image to run inference on. | +| `input-dir` | The directory of images to run inference on. | + + +## How to Run Inference +After the previous steps, you should be able to run the code using the following script that will pass all of the required parameters to the Infer API. + +From the `scala-package/examples/scripts/inferexample/objectdetector/` folder run: + +```bash +./run_ssd_example.sh ../models/resnet50_ssd/resnet50_ssd_model ../images/dog.jpg ../images +``` + +**Notes**: +* These are relative paths to this script. +* You may need to run `chmod +x run_ssd_example.sh` before running this script. + +The example should give expected output as shown below: +``` +Class: car +Probabilties: 0.99847263 +(Coord:,312.21335,72.0291,456.01443,150.66176) +Class: bicycle +Probabilties: 0.90473825 +(Coord:,155.95807,149.96362,383.8369,418.94513) +Class: dog +Probabilties: 0.8226818 +(Coord:,83.82353,179.13998,206.63783,476.7875) +``` +the outputs come from the the input image, with top3 predictions picked. + + +## Infer API Details +This example uses ObjectDetector class provided by MXNet's scala package Infer APIs. It provides methods to load the images, create NDArray out of Java BufferedImage and run prediction using Classifier and Predictor APIs. + + +## References +This documentation used the model and inference setup guide from the [MXNet Model Server SSD example](https://github.com/awslabs/mxnet-model-server/blob/master/examples/ssd/README.md). + + +## Next Steps + +Check out the following related tutorials and examples for the Infer API: + +* [Image Classification with the MXNet Scala Infer API](../imageclassifier/README.md) diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java new file mode 100644 index 000000000000..18f616b76185 --- /dev/null +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java @@ -0,0 +1,174 @@ +package org.apache.mxnetexamples.infer.javapi.objectdetector; + +import org.kohsuke.args4j.CmdLineParser; +import org.kohsuke.args4j.Option; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.mxnet.javaapi.*; +import org.apache.mxnet.infer.javaapi.ImageClassifier; +import org.apache.mxnet.infer.javaapi.ObjectDetector; + +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import javafx.util.Pair; + +import java.io.File; + +public class SSDClassifierExample { + @Option(name = "--model-path-prefix", usage = "the input model directory and prefix of the model") + private String modelPathPrefix = "/model/ssd_resnet50_512"; + @Option(name = "--input-image", usage = "the input image") + private String inputImagePath = "/images/dog.jpg"; + @Option(name = "--input-dir", usage = "the input batch of images directory") + private String inputImageDir = "/images/"; + + final static Logger logger = LoggerFactory.getLogger(SSDClassifierExample.class); + + static List>>> runObjectDetectionSingle(String modelPathPrefix, String inputImagePath, List context) { + Shape inputShape = new Shape(new int[] {1, 3, 512, 512}); + List inputDescriptors = new ArrayList(); + inputDescriptors.add(new DataDesc("data", inputShape, DType.Float32(), "NCHW")); + BufferedImage img = ImageClassifier.loadImageFromFile(inputImagePath); + ObjectDetector objDetector = new ObjectDetector(modelPathPrefix, inputDescriptors, context, 0); + return objDetector.imageObjectDetect(img, 3); + } + + static List>>>> runObjectDetectionBatch(String modelPathPrefix, String inputImageDir, List context) { + Shape inputShape = new Shape(new int[]{1, 3, 512, 512}); + List inputDescriptors = new ArrayList(); + inputDescriptors.add(new DataDesc("data", inputShape, DType.Float32(), "NCHW")); + ObjectDetector objDetector = new ObjectDetector(modelPathPrefix, inputDescriptors, context, 0); + + // Loading batch of images from the directory path + List> batchFiles = generateBatches(inputImageDir, 20); + List>>>> outputList = new ArrayList>>>>(); + + for (List batchFile : batchFiles) { + List imgList = ImageClassifier.loadInputBatch(batchFile); + // Running inference on batch of images loaded in previous step + List>>> tmp = objDetector.imageBatchObjectDetect(imgList, 5); + outputList.add(tmp); + } + return outputList; + } + + static List> generateBatches(String inputImageDirPath, int batchSize) { + File dir = new File(inputImageDirPath); + + List> output = new ArrayList>(); + List batch = new ArrayList(); + for (File imgFile : dir.listFiles()) { + batch.add(imgFile.getPath()); + if (batch.size() == batchSize) { + output.add(batch); + batch = new ArrayList(); + } + } + if (batch.size() > 0) { + output.add(batch); + } + return output; + } + + public static void main(String[] args) { + SSDClassifierExample inst = new SSDClassifierExample(); + CmdLineParser parser = new CmdLineParser(inst); + try { + parser.parseArgument(args); + } catch (Exception e) { + logger.error(e.getMessage(), e); + parser.printUsage(System.err); + System.exit(1); + } + + String mdprefixDir = inst.modelPathPrefix; + String imgPath = inst.inputImagePath; + String imgDir = inst.inputImageDir; + + if (!checkExist(Arrays.asList(mdprefixDir + "-symbol.json", imgDir, imgPath))) { + logger.error("Model or input image path does not exist"); + System.exit(1); + } + + List context = new ArrayList(); + if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && + Integer.valueOf(System.getenv("SCALA_TEST_ON_GPU")) == 1) { + context.add(Context.gpu()); + } else { + context.add(Context.cpu()); + } + + try { + Shape inputShape = new Shape(new int[] {1, 3, 512, 512}); + Shape outputShape = new Shape(new int[] {1, 6132, 6}); + + + int width = inputShape.get(2); + int height = inputShape.get(3); + String outputStr = "\n"; + + List>>> output = runObjectDetectionSingle(mdprefixDir, imgPath, context); + + for (List>> ele : output) { + for (Pair> i : ele) { + outputStr += "Class: " + i.getKey() + "\n"; + List arr = i.getValue(); + outputStr += "Probabilties: " + arr.get(0) + "\n"; + + List coord = Arrays.asList(arr.get(1) * width, arr.get(2) * height, arr.get(3) * width, arr.get(4) * height); + StringBuilder sb = new StringBuilder(); + for (float c: coord) { + sb.append(", ").append(c); + } + outputStr += "Coord:" + sb.substring(2)+ "\n"; + } + } + logger.info(outputStr); + + List>>>> outputList = runObjectDetectionBatch(mdprefixDir, imgDir, context); + + outputStr = "\n"; + int index = 0; + for (List>>> i: outputList) { + for (List>> j : i) { + outputStr += "*** Image " + (index + 1) + "***" + "\n"; + for (Pair> k : j) { + outputStr += "Class: " + k.getKey() + "\n"; + List arr = k.getValue(); + outputStr += "Probabilties: " + arr.get(0) + "\n"; + List coord = Arrays.asList(arr.get(1) * width, arr.get(2) * height, arr.get(3) * width, arr.get(4) * height); + + StringBuilder sb = new StringBuilder(); + for (float c : coord) { + sb.append(", ").append(c); + } + outputStr += "Coord:" + sb.substring(2) + "\n"; + } + index++; + } + } + logger.info(outputStr); + + } catch (Exception e) { + logger.error(e.getMessage(), e); + parser.printUsage(System.err); + System.exit(1); + } + System.exit(0); + } + + static Boolean checkExist(List arr) { + Boolean exist = true; + for (String item : arr) { + exist = new File(item).exists() && exist; + if (!exist) { + logger.error("Cannot find: " + item); + } + } + return exist; + } +} diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md index 69328a44bab6..bf4a44a76d00 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md @@ -31,7 +31,7 @@ You can download the files using the script `get_ssd_data.sh`. It will download From the `scala-package/examples/scripts/infer/imageclassifier/` folder run: ```bash -./get_resnet_data.sh +./get_ssd_data.sh ``` **Note**: You may need to run `chmod +x get_resnet_data.sh` before running this script. @@ -79,7 +79,7 @@ After the previous steps, you should be able to run the code using the following From the `scala-package/examples/scripts/inferexample/objectdetector/` folder run: ```bash -./run_ssd_example.sh ../models/resnet50_ssd_model ../images/dog.jpg ../images +./run_ssd_example.sh ../models/resnet50_ssd/resnet50_ssd_model ../images/dog.jpg ../images ``` **Notes**: diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala new file mode 100644 index 000000000000..5d6fcf382b47 --- /dev/null +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala @@ -0,0 +1,39 @@ +package org.apache.mxnet.infer.javaapi + +import java.awt.image.BufferedImage +import javafx.util.Pair + +import org.apache.mxnet.javaapi.{Context, DataDesc, NDArray} + +import scala.collection.JavaConverters +import scala.collection.JavaConverters._ + + +class ObjectDetector(val objectDetector: org.apache.mxnet.infer.ObjectDetector){ + + def this(modelPathPrefix: String, inputDescriptors: java.util.List[DataDesc], contexts: java.util.List[Context], epoch: Int) + = this { + val informationDesc = JavaConverters.asScalaIteratorConverter(inputDescriptors.iterator).asScala.toIndexedSeq map {a=>a: org.apache.mxnet.DataDesc} + val inContexts = (contexts.asScala.toList map {a => a: org.apache.mxnet.Context}).toArray + new org.apache.mxnet.infer.ObjectDetector(modelPathPrefix, informationDesc, inContexts, Some(epoch)) + } + + def imageObjectDetect(inputImage: BufferedImage, topK: Int): java.util.List[java.util.List[Pair[String, java.util.List[java.lang.Float]]]] = { + val ret = objectDetector.imageObjectDetect(inputImage, Some(topK)) + (ret map {a=> (a map {entry => new Pair[String, java.util.List[java.lang.Float]](entry._1, entry._2.map(f => Float.box(f)).toList.asJava)}).asJava }).asJava + } + + def objectDetectWithNDArray(input: java.util.List[NDArray], topK: Int): java.util.List[java.util.List[(String, java.util.List[java.lang.Float])]] = { + val ret = objectDetector.objectDetectWithNDArray(convert(input.asScala.toIndexedSeq), Some(topK)) + (ret map {a=> (a map {entry => (entry._1, entry._2.map(f => Float.box(f)).toList.asJava)}).asJava }).asJava + } + + def imageBatchObjectDetect(inputBatch: java.util.List[BufferedImage], topK: Int): + java.util.List[java.util.List[Pair[String, java.util.List[java.lang.Float]]]] = { + val ret = objectDetector.imageBatchObjectDetect(inputBatch.asScala, Some(topK)) + (ret map {a=> (a map {entry => new Pair[String, java.util.List[java.lang.Float]](entry._1, entry._2.map(f => Float.box(f)).toList.asJava)}).asJava }).asJava + } + + def convert[B, A <% B](l: IndexedSeq[A]): IndexedSeq[B] = l map { a => a: B } + +} diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala new file mode 100644 index 000000000000..8adf02a03513 --- /dev/null +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala @@ -0,0 +1,29 @@ +package org.apache.mxnet.infer.javaapi + +import org.apache.mxnet.infer.Predictor +import org.apache.mxnet.javaapi.{Context, DataDesc, NDArray} + +import scala.collection.JavaConverters +import scala.collection.JavaConverters._ + +class Predictor(val predictor: org.apache.mxnet.infer.Predictor){ + def this(modelPathPrevious:String, inputDescriptors: java.util.List[DataDesc], contexts: java.util.List[Context], epoch: Int) + = this { + val informationDesc = JavaConverters.asScalaIteratorConverter(inputDescriptors.iterator).asScala.toIndexedSeq map {a=>a: org.apache.mxnet.DataDesc} + val inContexts = (contexts.asScala.toList map {a => a: org.apache.mxnet.Context}).toArray + new org.apache.mxnet.infer.Predictor(modelPathPrevious, informationDesc, inContexts, Some(epoch)) + } + + def predict(input: java.util.List[java.util.List[Float]]): java.util.List[java.util.List[Float]] = { + val in = JavaConverters.asScalaIteratorConverter(input.iterator).asScala.toIndexedSeq + (predictor.predict(in map {a => a.asScala.toArray}) map {b => b.toList.asJava}).asJava + } + + def predictWithNDArray(input: java.util.List[NDArray]): java.util.List[NDArray] = { + val ret = predictor.predictWithNDArray(convert(JavaConverters.asScalaIteratorConverter(input.iterator).asScala.toIndexedSeq)) + // For some reason the implicit wasn't working here when trying to use convert. So did it this way. Needs to be figured out + (ret map {a => new NDArray(a)}).asJava + } + + def convert[B, A <% B](l: IndexedSeq[A]): IndexedSeq[B] = l map { a => a: B } +} From 540d42e3309013324aa536669df58d37a049d887 Mon Sep 17 00:00:00 2001 From: Ayres Date: Mon, 15 Oct 2018 15:14:17 -0700 Subject: [PATCH 02/17] Adding license to java files and fixing SSD example --- .../objectdetector/SSDClassifierExample.java | 17 +++++++++++ .../mxnet/infer/javaapi/ObjectDetector.scala | 28 +++++++++++++++++++ .../mxnet/infer/javaapi/Predictor.scala | 17 +++++++++++ 3 files changed, 62 insertions(+) diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java index 18f616b76185..7fb5371ac039 100644 --- a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.mxnetexamples.infer.javapi.objectdetector; import org.kohsuke.args4j.CmdLineParser; diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala index 5d6fcf382b47..656b527ba366 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.mxnet.infer.javaapi import java.awt.image.BufferedImage @@ -37,3 +54,14 @@ class ObjectDetector(val objectDetector: org.apache.mxnet.infer.ObjectDetector){ def convert[B, A <% B](l: IndexedSeq[A]): IndexedSeq[B] = l map { a => a: B } } + + +object ObjectDetector { + implicit def fromObjectDetector(OD: org.apache.mxnet.infer.ObjectDetector): ObjectDetector = new ObjectDetector(OD) + + implicit def toObjectDetector(jOD: ObjectDetector): org.apache.mxnet.infer.ObjectDetector = jOD.objectDetector + + def loadImageFromFile(inputImagePath: String): BufferedImage = { + org.apache.mxnet.infer.ImageClassifier.loadImageFromFile(inputImagePath) + } +} diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala index 8adf02a03513..e2d5643f281c 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.mxnet.infer.javaapi import org.apache.mxnet.infer.Predictor From 94559e7af8a7838c37ed8bfe59a88701616d92fd Mon Sep 17 00:00:00 2001 From: Ayres Date: Mon, 15 Oct 2018 15:21:20 -0700 Subject: [PATCH 03/17] Fixing SSD example to point to ObjectDetector instead of ImageClassifier --- .../infer/javapi/objectdetector/SSDClassifierExample.java | 5 ++--- .../org/apache/mxnet/infer/javaapi/ObjectDetector.scala | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java index 7fb5371ac039..8d47613639e5 100644 --- a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java @@ -24,7 +24,6 @@ import org.slf4j.LoggerFactory; import org.apache.mxnet.javaapi.*; -import org.apache.mxnet.infer.javaapi.ImageClassifier; import org.apache.mxnet.infer.javaapi.ObjectDetector; import java.awt.image.BufferedImage; @@ -49,7 +48,7 @@ static List>>> runObjectDetectionSingle(String mod Shape inputShape = new Shape(new int[] {1, 3, 512, 512}); List inputDescriptors = new ArrayList(); inputDescriptors.add(new DataDesc("data", inputShape, DType.Float32(), "NCHW")); - BufferedImage img = ImageClassifier.loadImageFromFile(inputImagePath); + BufferedImage img = ObjectDetector.loadImageFromFile(inputImagePath); ObjectDetector objDetector = new ObjectDetector(modelPathPrefix, inputDescriptors, context, 0); return objDetector.imageObjectDetect(img, 3); } @@ -65,7 +64,7 @@ static List>>>> runObjectDetectionBatch(Strin List>>>> outputList = new ArrayList>>>>(); for (List batchFile : batchFiles) { - List imgList = ImageClassifier.loadInputBatch(batchFile); + List imgList = ObjectDetector.loadInputBatch(batchFile); // Running inference on batch of images loaded in previous step List>>> tmp = objDetector.imageBatchObjectDetect(imgList, 5); outputList.add(tmp); diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala index 656b527ba366..583a96d3dd0a 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala @@ -64,4 +64,8 @@ object ObjectDetector { def loadImageFromFile(inputImagePath: String): BufferedImage = { org.apache.mxnet.infer.ImageClassifier.loadImageFromFile(inputImagePath) } + + def loadInputBatch(inputImagePaths: java.util.List[String]): java.util.List[BufferedImage] = { + org.apache.mxnet.infer.ImageClassifier.loadInputBatch(inputImagePaths.asScala.toList).toList.asJava + } } From c05efe5db834660de70fdb73c29e8d75c41c2ee4 Mon Sep 17 00:00:00 2001 From: Ayres Date: Mon, 15 Oct 2018 15:56:04 -0700 Subject: [PATCH 04/17] Make scripts for object detector independent to os and hw cpu/gpu --- .../infer/objectdetector/run_ssd_example.sh | 14 +++++++++++++- .../infer/objectdetector/run_ssd_java_example.sh | 14 +++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/scala-package/examples/scripts/infer/objectdetector/run_ssd_example.sh b/scala-package/examples/scripts/infer/objectdetector/run_ssd_example.sh index 8cea892b5809..adb8830de06e 100755 --- a/scala-package/examples/scripts/infer/objectdetector/run_ssd_example.sh +++ b/scala-package/examples/scripts/infer/objectdetector/run_ssd_example.sh @@ -17,9 +17,21 @@ # specific language governing permissions and limitations # under the License. +hw_type=cpu +if [[ $1 = gpu ]] +then + hw_type=gpu +fi + +platform=linux-x86_64 + +if [[ $OSTYPE = [darwin]* ]] +then + platform=osx-x86_64 +fi MXNET_ROOT=$(cd "$(dirname $0)/../../../../../"; pwd) -CLASS_PATH=$MXNET_ROOT/scala-package/assembly/osx-x86_64-cpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/*:$MXNET_ROOT/scala-package/infer/target/* +CLASS_PATH=$MXNET_ROOT/scala-package/assembly/$platform-$hw_type/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/*:$MXNET_ROOT/scala-package/infer/target/* # model dir and prefix MODEL_DIR=$1 diff --git a/scala-package/examples/scripts/infer/objectdetector/run_ssd_java_example.sh b/scala-package/examples/scripts/infer/objectdetector/run_ssd_java_example.sh index c7e58cebc513..11742889905a 100755 --- a/scala-package/examples/scripts/infer/objectdetector/run_ssd_java_example.sh +++ b/scala-package/examples/scripts/infer/objectdetector/run_ssd_java_example.sh @@ -17,9 +17,21 @@ # specific language governing permissions and limitations # under the License. +hw_type=cpu +if [[ $1 = gpu ]] +then + hw_type=gpu +fi + +platform=linux-x86_64 + +if [[ $OSTYPE = [darwin]* ]] +then + platform=osx-x86_64 +fi MXNET_ROOT=$(cd "$(dirname $0)/../../../../../"; pwd) -CLASS_PATH=$MXNET_ROOT/scala-package/assembly/osx-x86_64-cpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/*:$MXNET_ROOT/scala-package/infer/target/*:$MXNET_ROOT/scala-package/examples/src/main/scala/org/apache/mxnetexamples/api/java/infer/imageclassifier/* +CLASS_PATH=$MXNET_ROOT/scala-package/assembly/$platform-$hw_type/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/*:$MXNET_ROOT/scala-package/infer/target/*:$MXNET_ROOT/scala-package/examples/src/main/scala/org/apache/mxnetexamples/api/java/infer/imageclassifier/* # model dir and prefix MODEL_DIR=$1 From 20854425911e8c179124883e9b2f42d0f497e8d5 Mon Sep 17 00:00:00 2001 From: Ayres Date: Mon, 15 Oct 2018 16:08:31 -0700 Subject: [PATCH 05/17] Added API Docs to Java Inference API. Small fixes for PR --- .../mxnet/infer/javaapi/ObjectDetector.scala | 25 +++++++++++++++++ .../mxnet/infer/javaapi/Predictor.scala | 27 +++++++++++++++---- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala index 583a96d3dd0a..f927fe585cd8 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala @@ -35,16 +35,41 @@ class ObjectDetector(val objectDetector: org.apache.mxnet.infer.ObjectDetector){ new org.apache.mxnet.infer.ObjectDetector(modelPathPrefix, informationDesc, inContexts, Some(epoch)) } + /** + * Detects objects and returns bounding boxes with corresponding class/label + * + * @param inputImage Path prefix of the input image + * @param topK Number of result elements to return, sorted by probability + * @return List of list of tuples of + * (class, [probability, xmin, ymin, xmax, ymax]) + */ def imageObjectDetect(inputImage: BufferedImage, topK: Int): java.util.List[java.util.List[Pair[String, java.util.List[java.lang.Float]]]] = { val ret = objectDetector.imageObjectDetect(inputImage, Some(topK)) (ret map {a=> (a map {entry => new Pair[String, java.util.List[java.lang.Float]](entry._1, entry._2.map(f => Float.box(f)).toList.asJava)}).asJava }).asJava } + /** + * Takes input images as NDArrays. Useful when you want to perform multiple operations on + * the input array, or when you want to pass a batch of input images. + * + * @param input Indexed Sequence of NDArrays + * @param topK (Optional) How many top_k (sorting will be based on the last axis) + * elements to return. If not passed, returns all unsorted output. + * @return List of list of tuples of + * (class, [probability, xmin, ymin, xmax, ymax]) + */ def objectDetectWithNDArray(input: java.util.List[NDArray], topK: Int): java.util.List[java.util.List[(String, java.util.List[java.lang.Float])]] = { val ret = objectDetector.objectDetectWithNDArray(convert(input.asScala.toIndexedSeq), Some(topK)) (ret map {a=> (a map {entry => (entry._1, entry._2.map(f => Float.box(f)).toList.asJava)}).asJava }).asJava } + /** + * To classify batch of input images according to the provided model + * + * @param inputBatch Input array of buffered images + * @param topK Number of result elements to return, sorted by probability + * @return List of list of tuples of (class, probability) + */ def imageBatchObjectDetect(inputBatch: java.util.List[BufferedImage], topK: Int): java.util.List[java.util.List[Pair[String, java.util.List[java.lang.Float]]]] = { val ret = objectDetector.imageBatchObjectDetect(inputBatch.asScala, Some(topK)) diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala index e2d5643f281c..a3176fb024e8 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala @@ -17,30 +17,47 @@ package org.apache.mxnet.infer.javaapi -import org.apache.mxnet.infer.Predictor import org.apache.mxnet.javaapi.{Context, DataDesc, NDArray} import scala.collection.JavaConverters import scala.collection.JavaConverters._ class Predictor(val predictor: org.apache.mxnet.infer.Predictor){ - def this(modelPathPrevious:String, inputDescriptors: java.util.List[DataDesc], contexts: java.util.List[Context], epoch: Int) + def this(modelPathPrefix:String, inputDescriptors: java.util.List[DataDesc], contexts: java.util.List[Context], epoch: Int) = this { val informationDesc = JavaConverters.asScalaIteratorConverter(inputDescriptors.iterator).asScala.toIndexedSeq map {a=>a: org.apache.mxnet.DataDesc} val inContexts = (contexts.asScala.toList map {a => a: org.apache.mxnet.Context}).toArray - new org.apache.mxnet.infer.Predictor(modelPathPrevious, informationDesc, inContexts, Some(epoch)) + new org.apache.mxnet.infer.Predictor(modelPathPrefix, informationDesc, inContexts, Some(epoch)) } + + /** + * Takes input as List one dimensional arrays and creates the NDArray needed for inference + * The array will be reshaped based on the input descriptors. + * + * @param input: A List of a one-dimensional array. + An extra List is needed for when the model has more than one input. + * @return Indexed sequence array of outputs + */ def predict(input: java.util.List[java.util.List[Float]]): java.util.List[java.util.List[Float]] = { val in = JavaConverters.asScalaIteratorConverter(input.iterator).asScala.toIndexedSeq (predictor.predict(in map {a => a.asScala.toArray}) map {b => b.toList.asJava}).asJava } + + /** + * Predict using NDArray as input + * This method is useful when the input is a batch of data + * Note: User is responsible for managing allocation/deallocation of input/output NDArrays. + * + * @param input IndexedSequence NDArrays + * @return Output of predictions as NDArrays + */ def predictWithNDArray(input: java.util.List[NDArray]): java.util.List[NDArray] = { val ret = predictor.predictWithNDArray(convert(JavaConverters.asScalaIteratorConverter(input.iterator).asScala.toIndexedSeq)) - // For some reason the implicit wasn't working here when trying to use convert. So did it this way. Needs to be figured out + // TODO: For some reason the implicit wasn't working here when trying to use convert. So did it this way. Needs to be figured out (ret map {a => new NDArray(a)}).asJava } - def convert[B, A <% B](l: IndexedSeq[A]): IndexedSeq[B] = l map { a => a: B } + private def convert[B, A <% B](l: IndexedSeq[A]): IndexedSeq[B] = l map { a => a: B } } From 8394af273cf9786e1fdd23177ca05c6c261dfa5e Mon Sep 17 00:00:00 2001 From: Ayres Date: Mon, 15 Oct 2018 16:36:46 -0700 Subject: [PATCH 06/17] Cosmetic updates for API DOCS requested during PR --- .../main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala index a3176fb024e8..183b89df6f21 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala @@ -32,7 +32,7 @@ class Predictor(val predictor: org.apache.mxnet.infer.Predictor){ /** - * Takes input as List one dimensional arrays and creates the NDArray needed for inference + * Takes input as List of one dimensional arrays and creates the NDArray needed for inference * The array will be reshaped based on the input descriptors. * * @param input: A List of a one-dimensional array. @@ -50,7 +50,7 @@ class Predictor(val predictor: org.apache.mxnet.infer.Predictor){ * This method is useful when the input is a batch of data * Note: User is responsible for managing allocation/deallocation of input/output NDArrays. * - * @param input IndexedSequence NDArrays + * @param input List of NDArrays * @return Output of predictions as NDArrays */ def predictWithNDArray(input: java.util.List[NDArray]): java.util.List[NDArray] = { From 7122abaa77b4da4bcd887acd6467365d948a8008 Mon Sep 17 00:00:00 2001 From: Ayres Date: Mon, 15 Oct 2018 17:31:21 -0700 Subject: [PATCH 07/17] Attempt to fix the CI Javafx compiler issue --- scala-package/infer/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index e50100169328..285b2bd69f48 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -101,5 +101,12 @@ 1.10.19 test + + com.oracle + javafx + 2.2 + ${java.home}/lib/ext/jfxrt.jar + system + From 76eb0b884f879127d571bb4e5c23bd766d9742ff Mon Sep 17 00:00:00 2001 From: Ayres Date: Tue, 16 Oct 2018 11:32:13 -0700 Subject: [PATCH 08/17] Migrate from Javafx to apache commons for Pair implementation --- .../objectdetector/SSDClassifierExample.java | 24 +++++++++---------- scala-package/infer/pom.xml | 5 ++++ .../mxnet/infer/javaapi/ObjectDetector.scala | 10 ++++---- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java index 8d47613639e5..888bba70217d 100644 --- a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java @@ -30,7 +30,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import javafx.util.Pair; +import org.apache.commons.lang3.tuple.ImmutablePair; import java.io.File; @@ -44,7 +44,7 @@ public class SSDClassifierExample { final static Logger logger = LoggerFactory.getLogger(SSDClassifierExample.class); - static List>>> runObjectDetectionSingle(String modelPathPrefix, String inputImagePath, List context) { + static List>>> runObjectDetectionSingle(String modelPathPrefix, String inputImagePath, List context) { Shape inputShape = new Shape(new int[] {1, 3, 512, 512}); List inputDescriptors = new ArrayList(); inputDescriptors.add(new DataDesc("data", inputShape, DType.Float32(), "NCHW")); @@ -53,7 +53,7 @@ static List>>> runObjectDetectionSingle(String mod return objDetector.imageObjectDetect(img, 3); } - static List>>>> runObjectDetectionBatch(String modelPathPrefix, String inputImageDir, List context) { + static List>>>> runObjectDetectionBatch(String modelPathPrefix, String inputImageDir, List context) { Shape inputShape = new Shape(new int[]{1, 3, 512, 512}); List inputDescriptors = new ArrayList(); inputDescriptors.add(new DataDesc("data", inputShape, DType.Float32(), "NCHW")); @@ -61,12 +61,12 @@ static List>>>> runObjectDetectionBatch(Strin // Loading batch of images from the directory path List> batchFiles = generateBatches(inputImageDir, 20); - List>>>> outputList = new ArrayList>>>>(); + List>>>> outputList = new ArrayList>>>>(); for (List batchFile : batchFiles) { List imgList = ObjectDetector.loadInputBatch(batchFile); // Running inference on batch of images loaded in previous step - List>>> tmp = objDetector.imageBatchObjectDetect(imgList, 5); + List>>> tmp = objDetector.imageBatchObjectDetect(imgList, 5); outputList.add(tmp); } return outputList; @@ -127,10 +127,10 @@ public static void main(String[] args) { int height = inputShape.get(3); String outputStr = "\n"; - List>>> output = runObjectDetectionSingle(mdprefixDir, imgPath, context); + List>>> output = runObjectDetectionSingle(mdprefixDir, imgPath, context); - for (List>> ele : output) { - for (Pair> i : ele) { + for (List>> ele : output) { + for (ImmutablePair> i : ele) { outputStr += "Class: " + i.getKey() + "\n"; List arr = i.getValue(); outputStr += "Probabilties: " + arr.get(0) + "\n"; @@ -145,14 +145,14 @@ public static void main(String[] args) { } logger.info(outputStr); - List>>>> outputList = runObjectDetectionBatch(mdprefixDir, imgDir, context); + List>>>> outputList = runObjectDetectionBatch(mdprefixDir, imgDir, context); outputStr = "\n"; int index = 0; - for (List>>> i: outputList) { - for (List>> j : i) { + for (List>>> i: outputList) { + for (List>> j : i) { outputStr += "*** Image " + (index + 1) + "***" + "\n"; - for (Pair> k : j) { + for (ImmutablePair> k : j) { outputStr += "Class: " + k.getKey() + "\n"; List arr = k.getValue(); outputStr += "Probabilties: " + arr.get(0) + "\n"; diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index 285b2bd69f48..309f00cacf81 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -108,5 +108,10 @@ ${java.home}/lib/ext/jfxrt.jar system + + org.apache.commons + commons-lang3 + 3.0 + diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala index f927fe585cd8..d45f94ae4f80 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala @@ -18,7 +18,7 @@ package org.apache.mxnet.infer.javaapi import java.awt.image.BufferedImage -import javafx.util.Pair +import org.apache.commons.lang3.tuple.ImmutablePair import org.apache.mxnet.javaapi.{Context, DataDesc, NDArray} @@ -43,9 +43,9 @@ class ObjectDetector(val objectDetector: org.apache.mxnet.infer.ObjectDetector){ * @return List of list of tuples of * (class, [probability, xmin, ymin, xmax, ymax]) */ - def imageObjectDetect(inputImage: BufferedImage, topK: Int): java.util.List[java.util.List[Pair[String, java.util.List[java.lang.Float]]]] = { + def imageObjectDetect(inputImage: BufferedImage, topK: Int): java.util.List[java.util.List[ImmutablePair[String, java.util.List[java.lang.Float]]]] = { val ret = objectDetector.imageObjectDetect(inputImage, Some(topK)) - (ret map {a=> (a map {entry => new Pair[String, java.util.List[java.lang.Float]](entry._1, entry._2.map(f => Float.box(f)).toList.asJava)}).asJava }).asJava + (ret map {a=> (a map {entry => new ImmutablePair[String, java.util.List[java.lang.Float]](entry._1, entry._2.map(f => Float.box(f)).toList.asJava)}).asJava }).asJava } /** @@ -71,9 +71,9 @@ class ObjectDetector(val objectDetector: org.apache.mxnet.infer.ObjectDetector){ * @return List of list of tuples of (class, probability) */ def imageBatchObjectDetect(inputBatch: java.util.List[BufferedImage], topK: Int): - java.util.List[java.util.List[Pair[String, java.util.List[java.lang.Float]]]] = { + java.util.List[java.util.List[ImmutablePair[String, java.util.List[java.lang.Float]]]] = { val ret = objectDetector.imageBatchObjectDetect(inputBatch.asScala, Some(topK)) - (ret map {a=> (a map {entry => new Pair[String, java.util.List[java.lang.Float]](entry._1, entry._2.map(f => Float.box(f)).toList.asJava)}).asJava }).asJava + (ret map {a=> (a map {entry => new ImmutablePair[String, java.util.List[java.lang.Float]](entry._1, entry._2.map(f => Float.box(f)).toList.asJava)}).asJava }).asJava } def convert[B, A <% B](l: IndexedSeq[A]): IndexedSeq[B] = l map { a => a: B } From 2f8448d90fcd07271b859acf7fdbed9ee2541371 Mon Sep 17 00:00:00 2001 From: Ayres Date: Tue, 16 Oct 2018 13:06:02 -0700 Subject: [PATCH 09/17] Removing javafx from pom file --- scala-package/infer/pom.xml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index 309f00cacf81..76a5d8ec2e47 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -101,13 +101,6 @@ 1.10.19 test - - com.oracle - javafx - 2.2 - ${java.home}/lib/ext/jfxrt.jar - system - org.apache.commons commons-lang3 From a9d93b8708d0f0f6ccc1d4d2a47005494517f218 Mon Sep 17 00:00:00 2001 From: Ayres Date: Tue, 16 Oct 2018 14:39:34 -0700 Subject: [PATCH 10/17] Fixes to appease the ScalaStyle deity --- .../objectdetector/SSDClassifierExample.java | 35 +++++++++----- .../mxnet/infer/javaapi/ObjectDetector.scala | 47 +++++++++++++------ .../mxnet/infer/javaapi/Predictor.scala | 18 ++++--- 3 files changed, 68 insertions(+), 32 deletions(-) diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java index 888bba70217d..93a8bc2c85fe 100644 --- a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java @@ -26,7 +26,10 @@ import org.apache.mxnet.javaapi.*; import org.apache.mxnet.infer.javaapi.ObjectDetector; +// scalastyle:off import java.awt.image.BufferedImage; +// scalastyle:on + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -35,7 +38,7 @@ import java.io.File; public class SSDClassifierExample { - @Option(name = "--model-path-prefix", usage = "the input model directory and prefix of the model") + @Option(name = "--model-path-prefix", usage = "input model directory and prefix of the model") private String modelPathPrefix = "/model/ssd_resnet50_512"; @Option(name = "--input-image", usage = "the input image") private String inputImagePath = "/images/dog.jpg"; @@ -44,29 +47,33 @@ public class SSDClassifierExample { final static Logger logger = LoggerFactory.getLogger(SSDClassifierExample.class); - static List>>> runObjectDetectionSingle(String modelPathPrefix, String inputImagePath, List context) { + static List>>> + runObjectDetectionSingle(String modelPathPrefix, String inputImagePath, List context) { Shape inputShape = new Shape(new int[] {1, 3, 512, 512}); List inputDescriptors = new ArrayList(); inputDescriptors.add(new DataDesc("data", inputShape, DType.Float32(), "NCHW")); BufferedImage img = ObjectDetector.loadImageFromFile(inputImagePath); - ObjectDetector objDetector = new ObjectDetector(modelPathPrefix, inputDescriptors, context, 0); - return objDetector.imageObjectDetect(img, 3); + ObjectDetector objDet = new ObjectDetector(modelPathPrefix, inputDescriptors, context, 0); + return objDet.imageObjectDetect(img, 3); } - static List>>>> runObjectDetectionBatch(String modelPathPrefix, String inputImageDir, List context) { + static List>>>> + runObjectDetectionBatch(String modelPathPrefix, String inputImageDir, List context) { Shape inputShape = new Shape(new int[]{1, 3, 512, 512}); List inputDescriptors = new ArrayList(); inputDescriptors.add(new DataDesc("data", inputShape, DType.Float32(), "NCHW")); - ObjectDetector objDetector = new ObjectDetector(modelPathPrefix, inputDescriptors, context, 0); + ObjectDetector objDet = new ObjectDetector(modelPathPrefix, inputDescriptors, context, 0); // Loading batch of images from the directory path List> batchFiles = generateBatches(inputImageDir, 20); - List>>>> outputList = new ArrayList>>>>(); + List>>>> outputList + = new ArrayList>>>>(); for (List batchFile : batchFiles) { List imgList = ObjectDetector.loadInputBatch(batchFile); // Running inference on batch of images loaded in previous step - List>>> tmp = objDetector.imageBatchObjectDetect(imgList, 5); + List>>> tmp + = objDet.imageBatchObjectDetect(imgList, 5); outputList.add(tmp); } return outputList; @@ -127,7 +134,8 @@ public static void main(String[] args) { int height = inputShape.get(3); String outputStr = "\n"; - List>>> output = runObjectDetectionSingle(mdprefixDir, imgPath, context); + List>>> output + = runObjectDetectionSingle(mdprefixDir, imgPath, context); for (List>> ele : output) { for (ImmutablePair> i : ele) { @@ -135,7 +143,8 @@ public static void main(String[] args) { List arr = i.getValue(); outputStr += "Probabilties: " + arr.get(0) + "\n"; - List coord = Arrays.asList(arr.get(1) * width, arr.get(2) * height, arr.get(3) * width, arr.get(4) * height); + List coord = Arrays.asList(arr.get(1) * width, + arr.get(2) * height, arr.get(3) * width, arr.get(4) * height); StringBuilder sb = new StringBuilder(); for (float c: coord) { sb.append(", ").append(c); @@ -145,7 +154,8 @@ public static void main(String[] args) { } logger.info(outputStr); - List>>>> outputList = runObjectDetectionBatch(mdprefixDir, imgDir, context); + List>>>> outputList = + runObjectDetectionBatch(mdprefixDir, imgDir, context); outputStr = "\n"; int index = 0; @@ -156,7 +166,8 @@ public static void main(String[] args) { outputStr += "Class: " + k.getKey() + "\n"; List arr = k.getValue(); outputStr += "Probabilties: " + arr.get(0) + "\n"; - List coord = Arrays.asList(arr.get(1) * width, arr.get(2) * height, arr.get(3) * width, arr.get(4) * height); + List coord = Arrays.asList(arr.get(1) * width, + arr.get(2) * height, arr.get(3) * width, arr.get(4) * height); StringBuilder sb = new StringBuilder(); for (float c : coord) { diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala index d45f94ae4f80..5485ab078e02 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala @@ -17,7 +17,10 @@ package org.apache.mxnet.infer.javaapi +// scalastyle:off import java.awt.image.BufferedImage +// scalastyle:on + import org.apache.commons.lang3.tuple.ImmutablePair import org.apache.mxnet.javaapi.{Context, DataDesc, NDArray} @@ -26,13 +29,17 @@ import scala.collection.JavaConverters import scala.collection.JavaConverters._ -class ObjectDetector(val objectDetector: org.apache.mxnet.infer.ObjectDetector){ +class ObjectDetector(val objDetector: org.apache.mxnet.infer.ObjectDetector){ - def this(modelPathPrefix: String, inputDescriptors: java.util.List[DataDesc], contexts: java.util.List[Context], epoch: Int) + def this(modelPathPrefix: String, inputDescriptors: java.util.List[DataDesc], contexts: + java.util.List[Context], epoch: Int) = this { - val informationDesc = JavaConverters.asScalaIteratorConverter(inputDescriptors.iterator).asScala.toIndexedSeq map {a=>a: org.apache.mxnet.DataDesc} + val informationDesc = JavaConverters.asScalaIteratorConverter(inputDescriptors.iterator) + .asScala.toIndexedSeq map {a => a: org.apache.mxnet.DataDesc} val inContexts = (contexts.asScala.toList map {a => a: org.apache.mxnet.Context}).toArray + // scalastyle:off new org.apache.mxnet.infer.ObjectDetector(modelPathPrefix, informationDesc, inContexts, Some(epoch)) + // scalastyle:on } /** @@ -43,9 +50,13 @@ class ObjectDetector(val objectDetector: org.apache.mxnet.infer.ObjectDetector){ * @return List of list of tuples of * (class, [probability, xmin, ymin, xmax, ymax]) */ - def imageObjectDetect(inputImage: BufferedImage, topK: Int): java.util.List[java.util.List[ImmutablePair[String, java.util.List[java.lang.Float]]]] = { - val ret = objectDetector.imageObjectDetect(inputImage, Some(topK)) - (ret map {a=> (a map {entry => new ImmutablePair[String, java.util.List[java.lang.Float]](entry._1, entry._2.map(f => Float.box(f)).toList.asJava)}).asJava }).asJava + def imageObjectDetect(inputImage: BufferedImage, topK: Int): + java.util.List[java.util.List[ImmutablePair[String, java.util.List[java.lang.Float]]]] = { + val ret = objDetector.imageObjectDetect(inputImage, Some(topK)) + (ret map {a => (a map {entry => + new ImmutablePair[String, java.util.List[java.lang.Float]](entry._1, + entry._2.map(f => Float.box(f)).toList.asJava)}).asJava + }).asJava } /** @@ -58,9 +69,11 @@ class ObjectDetector(val objectDetector: org.apache.mxnet.infer.ObjectDetector){ * @return List of list of tuples of * (class, [probability, xmin, ymin, xmax, ymax]) */ - def objectDetectWithNDArray(input: java.util.List[NDArray], topK: Int): java.util.List[java.util.List[(String, java.util.List[java.lang.Float])]] = { - val ret = objectDetector.objectDetectWithNDArray(convert(input.asScala.toIndexedSeq), Some(topK)) - (ret map {a=> (a map {entry => (entry._1, entry._2.map(f => Float.box(f)).toList.asJava)}).asJava }).asJava + def objectDetectWithNDArray(input: java.util.List[NDArray], topK: Int): + java.util.List[java.util.List[(String, java.util.List[java.lang.Float])]] = { + val ret = objDetector.objectDetectWithNDArray(convert(input.asScala.toIndexedSeq), Some(topK)) + (ret map {a => (a map {entry => + (entry._1, entry._2.map(f => Float.box(f)).toList.asJava)}).asJava }).asJava } /** @@ -72,8 +85,11 @@ class ObjectDetector(val objectDetector: org.apache.mxnet.infer.ObjectDetector){ */ def imageBatchObjectDetect(inputBatch: java.util.List[BufferedImage], topK: Int): java.util.List[java.util.List[ImmutablePair[String, java.util.List[java.lang.Float]]]] = { - val ret = objectDetector.imageBatchObjectDetect(inputBatch.asScala, Some(topK)) - (ret map {a=> (a map {entry => new ImmutablePair[String, java.util.List[java.lang.Float]](entry._1, entry._2.map(f => Float.box(f)).toList.asJava)}).asJava }).asJava + val ret = objDetector.imageBatchObjectDetect(inputBatch.asScala, Some(topK)) + (ret map {a => (a map {entry => + new ImmutablePair[String, java.util.List[java.lang.Float]](entry._1, + entry._2.map(f => Float.box(f)).toList.asJava) + }).asJava }).asJava } def convert[B, A <% B](l: IndexedSeq[A]): IndexedSeq[B] = l map { a => a: B } @@ -82,15 +98,18 @@ class ObjectDetector(val objectDetector: org.apache.mxnet.infer.ObjectDetector){ object ObjectDetector { - implicit def fromObjectDetector(OD: org.apache.mxnet.infer.ObjectDetector): ObjectDetector = new ObjectDetector(OD) + implicit def fromObjectDetector(OD: org.apache.mxnet.infer.ObjectDetector): + ObjectDetector = new ObjectDetector(OD) - implicit def toObjectDetector(jOD: ObjectDetector): org.apache.mxnet.infer.ObjectDetector = jOD.objectDetector + implicit def toObjectDetector(jOD: ObjectDetector): + org.apache.mxnet.infer.ObjectDetector = jOD.objDetector def loadImageFromFile(inputImagePath: String): BufferedImage = { org.apache.mxnet.infer.ImageClassifier.loadImageFromFile(inputImagePath) } def loadInputBatch(inputImagePaths: java.util.List[String]): java.util.List[BufferedImage] = { - org.apache.mxnet.infer.ImageClassifier.loadInputBatch(inputImagePaths.asScala.toList).toList.asJava + org.apache.mxnet.infer.ImageClassifier + .loadInputBatch(inputImagePaths.asScala.toList).toList.asJava } } diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala index 183b89df6f21..26ccd06cf466 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/Predictor.scala @@ -23,9 +23,11 @@ import scala.collection.JavaConverters import scala.collection.JavaConverters._ class Predictor(val predictor: org.apache.mxnet.infer.Predictor){ - def this(modelPathPrefix:String, inputDescriptors: java.util.List[DataDesc], contexts: java.util.List[Context], epoch: Int) + def this(modelPathPrefix: String, inputDescriptors: java.util.List[DataDesc], + contexts: java.util.List[Context], epoch: Int) = this { - val informationDesc = JavaConverters.asScalaIteratorConverter(inputDescriptors.iterator).asScala.toIndexedSeq map {a=>a: org.apache.mxnet.DataDesc} + val informationDesc = JavaConverters.asScalaIteratorConverter(inputDescriptors.iterator) + .asScala.toIndexedSeq map {a => a: org.apache.mxnet.DataDesc} val inContexts = (contexts.asScala.toList map {a => a: org.apache.mxnet.Context}).toArray new org.apache.mxnet.infer.Predictor(modelPathPrefix, informationDesc, inContexts, Some(epoch)) } @@ -39,7 +41,8 @@ class Predictor(val predictor: org.apache.mxnet.infer.Predictor){ An extra List is needed for when the model has more than one input. * @return Indexed sequence array of outputs */ - def predict(input: java.util.List[java.util.List[Float]]): java.util.List[java.util.List[Float]] = { + def predict(input: java.util.List[java.util.List[Float]]): + java.util.List[java.util.List[Float]] = { val in = JavaConverters.asScalaIteratorConverter(input.iterator).asScala.toIndexedSeq (predictor.predict(in map {a => a.asScala.toArray}) map {b => b.toList.asJava}).asJava } @@ -53,9 +56,12 @@ class Predictor(val predictor: org.apache.mxnet.infer.Predictor){ * @param input List of NDArrays * @return Output of predictions as NDArrays */ - def predictWithNDArray(input: java.util.List[NDArray]): java.util.List[NDArray] = { - val ret = predictor.predictWithNDArray(convert(JavaConverters.asScalaIteratorConverter(input.iterator).asScala.toIndexedSeq)) - // TODO: For some reason the implicit wasn't working here when trying to use convert. So did it this way. Needs to be figured out + def predictWithNDArray(input: java.util.List[NDArray]): + java.util.List[NDArray] = { + val ret = predictor.predictWithNDArray(convert(JavaConverters + .asScalaIteratorConverter(input.iterator).asScala.toIndexedSeq)) + // TODO: For some reason the implicit wasn't working here when trying to use convert. + // So did it this way. Needs to be figured out (ret map {a => new NDArray(a)}).asJava } From 8d99eec3932299a6b56b77cf32b338fafac9657e Mon Sep 17 00:00:00 2001 From: Ayres Date: Wed, 17 Oct 2018 10:05:58 -0700 Subject: [PATCH 11/17] Minor fix in SSD script and Readme --- .../scripts/infer/objectdetector/run_ssd_java_example.sh | 2 +- .../apache/mxnetexamples/infer/javapi/objectdetector/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scala-package/examples/scripts/infer/objectdetector/run_ssd_java_example.sh b/scala-package/examples/scripts/infer/objectdetector/run_ssd_java_example.sh index 11742889905a..f444a3a59af7 100755 --- a/scala-package/examples/scripts/infer/objectdetector/run_ssd_java_example.sh +++ b/scala-package/examples/scripts/infer/objectdetector/run_ssd_java_example.sh @@ -18,7 +18,7 @@ # under the License. hw_type=cpu -if [[ $1 = gpu ]] +if [[ $4 = gpu ]] then hw_type=gpu fi diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/README.md b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/README.md index bf4a44a76d00..63b9f929a82e 100644 --- a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/README.md +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/README.md @@ -79,7 +79,7 @@ After the previous steps, you should be able to run the code using the following From the `scala-package/examples/scripts/inferexample/objectdetector/` folder run: ```bash -./run_ssd_example.sh ../models/resnet50_ssd/resnet50_ssd_model ../images/dog.jpg ../images +./run_ssd_example.sh ../models/resnet50_ssd/resnet50_ssd/resnet50_ssd_model ../images/dog.jpg ../images ``` **Notes**: From 91c9beeff08078c17d133fc044d7d117745ed91e Mon Sep 17 00:00:00 2001 From: Ayres Date: Wed, 17 Oct 2018 12:13:50 -0700 Subject: [PATCH 12/17] Added ObjectDetectorOutput which is a POJO for Object Detector to simplify the return type --- .../objectdetector/SSDClassifierExample.java | 43 +++++++++---------- .../mxnet/infer/javaapi/ObjectDetector.scala | 19 +++----- .../infer/javaapi/ObjectDetectorOutput.scala | 23 ++++++++++ 3 files changed, 50 insertions(+), 35 deletions(-) create mode 100644 scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java index 93a8bc2c85fe..0881b5ca9033 100644 --- a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java @@ -17,6 +17,7 @@ package org.apache.mxnetexamples.infer.javapi.objectdetector; +import org.apache.mxnet.infer.javaapi.ObjectDetectorOutput; import org.kohsuke.args4j.CmdLineParser; import org.kohsuke.args4j.Option; @@ -47,7 +48,7 @@ public class SSDClassifierExample { final static Logger logger = LoggerFactory.getLogger(SSDClassifierExample.class); - static List>>> + static List> runObjectDetectionSingle(String modelPathPrefix, String inputImagePath, List context) { Shape inputShape = new Shape(new int[] {1, 3, 512, 512}); List inputDescriptors = new ArrayList(); @@ -57,7 +58,7 @@ public class SSDClassifierExample { return objDet.imageObjectDetect(img, 3); } - static List>>>> + static List>> runObjectDetectionBatch(String modelPathPrefix, String inputImageDir, List context) { Shape inputShape = new Shape(new int[]{1, 3, 512, 512}); List inputDescriptors = new ArrayList(); @@ -66,13 +67,13 @@ public class SSDClassifierExample { // Loading batch of images from the directory path List> batchFiles = generateBatches(inputImageDir, 20); - List>>>> outputList - = new ArrayList>>>>(); + List>> outputList + = new ArrayList>>(); for (List batchFile : batchFiles) { List imgList = ObjectDetector.loadInputBatch(batchFile); // Running inference on batch of images loaded in previous step - List>>> tmp + List> tmp = objDet.imageBatchObjectDetect(imgList, 5); outputList.add(tmp); } @@ -134,17 +135,16 @@ public static void main(String[] args) { int height = inputShape.get(3); String outputStr = "\n"; - List>>> output + List> output = runObjectDetectionSingle(mdprefixDir, imgPath, context); - for (List>> ele : output) { - for (ImmutablePair> i : ele) { - outputStr += "Class: " + i.getKey() + "\n"; - List arr = i.getValue(); - outputStr += "Probabilties: " + arr.get(0) + "\n"; + for (List ele : output) { + for (ObjectDetectorOutput i : ele) { + outputStr += "Class: " + i.getClassName() + "\n"; + outputStr += "Probabilties: " + i.getProbability() + "\n"; - List coord = Arrays.asList(arr.get(1) * width, - arr.get(2) * height, arr.get(3) * width, arr.get(4) * height); + List coord = Arrays.asList(i.getXMin() * width, + i.getXMax() * height, i.getYMin() * width, i.getYMax() * height); StringBuilder sb = new StringBuilder(); for (float c: coord) { sb.append(", ").append(c); @@ -154,20 +154,19 @@ public static void main(String[] args) { } logger.info(outputStr); - List>>>> outputList = + List>> outputList = runObjectDetectionBatch(mdprefixDir, imgDir, context); outputStr = "\n"; int index = 0; - for (List>>> i: outputList) { - for (List>> j : i) { + for (List> i: outputList) { + for (List j : i) { outputStr += "*** Image " + (index + 1) + "***" + "\n"; - for (ImmutablePair> k : j) { - outputStr += "Class: " + k.getKey() + "\n"; - List arr = k.getValue(); - outputStr += "Probabilties: " + arr.get(0) + "\n"; - List coord = Arrays.asList(arr.get(1) * width, - arr.get(2) * height, arr.get(3) * width, arr.get(4) * height); + for (ObjectDetectorOutput k : j) { + outputStr += "Class: " + k.getClassName() + "\n"; + outputStr += "Probabilties: " + k.getProbability() + "\n"; + List coord = Arrays.asList(k.getXMin() * width, + k.getXMax() * height, k.getYMin() * width, k.getYMax() * height); StringBuilder sb = new StringBuilder(); for (float c : coord) { diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala index 5485ab078e02..aaa9f35ba8aa 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala @@ -51,12 +51,9 @@ class ObjectDetector(val objDetector: org.apache.mxnet.infer.ObjectDetector){ * (class, [probability, xmin, ymin, xmax, ymax]) */ def imageObjectDetect(inputImage: BufferedImage, topK: Int): - java.util.List[java.util.List[ImmutablePair[String, java.util.List[java.lang.Float]]]] = { + java.util.List[java.util.List[ObjectDetectorOutput]] = { val ret = objDetector.imageObjectDetect(inputImage, Some(topK)) - (ret map {a => (a map {entry => - new ImmutablePair[String, java.util.List[java.lang.Float]](entry._1, - entry._2.map(f => Float.box(f)).toList.asJava)}).asJava - }).asJava + (ret map {a => (a map {e => new ObjectDetectorOutput(e._1, e._2)}).asJava}).asJava } /** @@ -70,10 +67,9 @@ class ObjectDetector(val objDetector: org.apache.mxnet.infer.ObjectDetector){ * (class, [probability, xmin, ymin, xmax, ymax]) */ def objectDetectWithNDArray(input: java.util.List[NDArray], topK: Int): - java.util.List[java.util.List[(String, java.util.List[java.lang.Float])]] = { + java.util.List[java.util.List[ObjectDetectorOutput]] = { val ret = objDetector.objectDetectWithNDArray(convert(input.asScala.toIndexedSeq), Some(topK)) - (ret map {a => (a map {entry => - (entry._1, entry._2.map(f => Float.box(f)).toList.asJava)}).asJava }).asJava + (ret map {a => (a map {e => new ObjectDetectorOutput(e._1, e._2)}).asJava}).asJava } /** @@ -84,12 +80,9 @@ class ObjectDetector(val objDetector: org.apache.mxnet.infer.ObjectDetector){ * @return List of list of tuples of (class, probability) */ def imageBatchObjectDetect(inputBatch: java.util.List[BufferedImage], topK: Int): - java.util.List[java.util.List[ImmutablePair[String, java.util.List[java.lang.Float]]]] = { + java.util.List[java.util.List[ObjectDetectorOutput]] = { val ret = objDetector.imageBatchObjectDetect(inputBatch.asScala, Some(topK)) - (ret map {a => (a map {entry => - new ImmutablePair[String, java.util.List[java.lang.Float]](entry._1, - entry._2.map(f => Float.box(f)).toList.asJava) - }).asJava }).asJava + (ret map {a => (a map {e => new ObjectDetectorOutput(e._1, e._2)}).asJava}).asJava } def convert[B, A <% B](l: IndexedSeq[A]): IndexedSeq[B] = l map { a => a: B } diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala new file mode 100644 index 000000000000..36639c188ded --- /dev/null +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala @@ -0,0 +1,23 @@ +package org.apache.mxnet.infer.javaapi + +class ObjectDetectorOutput (className: String, args: Array[Float]){ + + private val probability = args(0) + private val xMin = args(1) + private val xMax = args(2) + private val yMin = args(3) + private val yMax = args(4) + + def getClassName = className + + def getProbability = probability + + def getXMin = xMin + + def getXMax = xMax + + def getYMin = yMin + + def getYMax = yMax + +} From eebe5f955f2f1f75332f5862acb271c982e7bfa0 Mon Sep 17 00:00:00 2001 From: Ayres Date: Wed, 17 Oct 2018 12:18:01 -0700 Subject: [PATCH 13/17] Removing Apache Commons Immutable Pair --- .../infer/javapi/objectdetector/SSDClassifierExample.java | 1 - scala-package/infer/pom.xml | 5 ----- .../org/apache/mxnet/infer/javaapi/ObjectDetector.scala | 2 -- 3 files changed, 8 deletions(-) diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java index 0881b5ca9033..13f9d2d9a3e5 100644 --- a/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/infer/javapi/objectdetector/SSDClassifierExample.java @@ -34,7 +34,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import org.apache.commons.lang3.tuple.ImmutablePair; import java.io.File; diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index 76a5d8ec2e47..e50100169328 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -101,10 +101,5 @@ 1.10.19 test - - org.apache.commons - commons-lang3 - 3.0 - diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala index aaa9f35ba8aa..6cd3df6b896b 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetector.scala @@ -21,8 +21,6 @@ package org.apache.mxnet.infer.javaapi import java.awt.image.BufferedImage // scalastyle:on -import org.apache.commons.lang3.tuple.ImmutablePair - import org.apache.mxnet.javaapi.{Context, DataDesc, NDArray} import scala.collection.JavaConverters From 2ea2142a5d2c846532a38370647363600b18f9c1 Mon Sep 17 00:00:00 2001 From: Ayres Date: Wed, 17 Oct 2018 12:26:26 -0700 Subject: [PATCH 14/17] Adding license to new file --- .../infer/javaapi/ObjectDetectorOutput.scala | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala index 36639c188ded..b7f43238826c 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.mxnet.infer.javaapi class ObjectDetectorOutput (className: String, args: Array[Float]){ From 0400dfefd279e70ad44ec78e7c526073a62d679a Mon Sep 17 00:00:00 2001 From: Ayres Date: Wed, 17 Oct 2018 13:13:13 -0700 Subject: [PATCH 15/17] Minor style fixes --- .../mxnet/infer/javaapi/ObjectDetectorOutput.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala index b7f43238826c..0d4f422048f2 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala @@ -25,16 +25,16 @@ class ObjectDetectorOutput (className: String, args: Array[Float]){ private val yMin = args(3) private val yMax = args(4) - def getClassName = className + def getClassName: String = className - def getProbability = probability + def getProbability: Float = probability - def getXMin = xMin + def getXMin: Float= xMin - def getXMax = xMax + def getXMax: Float = xMax - def getYMin = yMin + def getYMin: Float = yMin - def getYMax = yMax + def getYMax: Float = yMax } From d1cd19b062f7cf68b03f230c9e74c39c5ab24532 Mon Sep 17 00:00:00 2001 From: Ayres Date: Wed, 17 Oct 2018 13:46:58 -0700 Subject: [PATCH 16/17] minor style fix --- .../org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala index 0d4f422048f2..797bddd0d3ba 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala @@ -29,7 +29,7 @@ class ObjectDetectorOutput (className: String, args: Array[Float]){ def getProbability: Float = probability - def getXMin: Float= xMin + def getXMin: Float = xMin def getXMax: Float = xMax From 126bd4612283f24e92d44165f40d5f94e279530c Mon Sep 17 00:00:00 2001 From: Ayres Date: Thu, 18 Oct 2018 16:09:33 -0700 Subject: [PATCH 17/17] Updating to be in scala style and not explicitly declare some unnecessary variables --- .../infer/javaapi/ObjectDetectorOutput.scala | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala index 797bddd0d3ba..13369c8fcef5 100644 --- a/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala +++ b/scala-package/infer/src/main/scala/org/apache/mxnet/infer/javaapi/ObjectDetectorOutput.scala @@ -19,22 +19,16 @@ package org.apache.mxnet.infer.javaapi class ObjectDetectorOutput (className: String, args: Array[Float]){ - private val probability = args(0) - private val xMin = args(1) - private val xMax = args(2) - private val yMin = args(3) - private val yMax = args(4) - def getClassName: String = className - def getProbability: Float = probability + def getProbability: Float = args(0) - def getXMin: Float = xMin + def getXMin: Float = args(1) - def getXMax: Float = xMax + def getXMax: Float = args(2) - def getYMin: Float = yMin + def getYMin: Float = args(3) - def getYMax: Float = yMax + def getYMax: Float = args(4) }