Rebase from master

Author: karllessard

tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/NnOps.java

@@ -18,7 +18,6 @@
 package org.tensorflow.op;
 
 import java.util.List;
-import org.tensorflow.DataType;
 import org.tensorflow.Operand;
 import org.tensorflow.op.nn.AvgPool;
 import org.tensorflow.op.nn.AvgPool3d;
@@ -84,6 +83,7 @@
 import org.tensorflow.op.nn.Relu;
 import org.tensorflow.op.nn.Relu6;
 import org.tensorflow.op.nn.Selu;
+import org.tensorflow.op.nn.SigmoidCrossEntropyWithLogits;
 import org.tensorflow.op.nn.Softmax;
 import org.tensorflow.op.nn.SoftmaxCrossEntropyWithLogits;
 import org.tensorflow.op.nn.Softsign;
@@ -103,10 +103,13 @@
  * @see {@link Ops}
  */
 public final class NnOps {
+  public final NnRawOps raw;
+
   private final Scope scope;
 
   NnOps(Scope scope) {
     this.scope = scope;
+    raw = new NnRawOps(scope);
   }
 
   /**
@@ -1342,7 +1345,7 @@ public <T extends TNumber> MaxPool3d<T> maxPool3d(Operand<T> input, List<Long> k
   }
 
   /**
-   * Computes gradients of max pooling function.
+   * Computes gradients of 3D max pooling function.
    *
    * @param <U> data type for {@code output()} output
    * @param origInput The original input tensor.
@@ -1767,6 +1770,56 @@ public <T extends TNumber> Selu<T> selu(Operand<T> features) {
     return Selu.create(scope, features);
   }
 
+  /**
+   * Computes sigmoid cross entropy given <code>logits</code>.
+   *
+   *  <p>Measures the probability error in discrete classification tasks in which each class is
+   *  independent and not mutually exclusive. For instance, one could perform multilabel
+   *  classification where a picture can contain both an elephant and a dog at the same time.
+   *
+   *  <p>For brevity, let <code>x = logits</code>, <code>z = labels</code>. The logistic loss in
+   *  pseudo-code is
+   *
+   *  <pre>
+   *  z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
+   *   = z * -log(1 / (1 + exp(-x))) + (1 - z) * -log(exp(-x) / (1 + exp(-x)))
+   *   = z * log(1 + exp(-x)) + (1 - z) * (-log(exp(-x)) + log(1 + exp(-x)))
+   *   = z * log(1 + exp(-x)) + (1 - z) * (x + log(1 + exp(-x))
+   *   = (1 - z) * x + log(1 + exp(-x))
+   *   = x - x * z + log(1 + exp(-x))
+   *  </pre>
+   *
+   *  <p>For <code>x < 0</code>, to avoid overflow in <code>exp(-x)</code>, we reformulate the above
+   *
+   *  <pre>
+   *  x - x * z + log(1 + exp(-x))
+   *   = log(exp(x)) - x * z + log(1 + exp(-x))
+   *   = - x * z + log(1 + exp(x))
+   *  </pre>
+   *
+   *  <p>Hence, to ensure stability and avoid overflow, the implementation uses this equivalent
+   *  formulation
+   *
+   *  <pre>
+   *    max(x, 0) - x * z + log(1 + exp(-abs(x)))
+   *  </pre>
+   *
+   *  <p></ode>logits</code> and <code>labels</code> must have the same type and shape.
+   *
+   *  <p>
+   *
+   * @param scope The TensorFlow scope
+   * @param labels the labels
+   * @param logits the logits of type float32 or float64
+   * @param <T> the type of labels and logits
+   * @return the component-wise logistic losses.
+   * @throws IllegalArgumentException if logits' and labels' do not have the same shape
+   */
+  public <T extends TNumber> Operand<T> sigmoidCrossEntropyWithLogits(Operand<T> labels,
+      Operand<T> logits) {
+    return SigmoidCrossEntropyWithLogits.sigmoidCrossEntropyWithLogits(scope, labels, logits);
+  }
+
   /**
    * Computes softmax activations.
    *  <p>
@@ -1783,20 +1836,54 @@ public <T extends TNumber> Softmax<T> softmax(Operand<T> logits) {
   }
 
   /**
-   * Computes softmax cross entropy cost and gradients to backpropagate.
-   *  <p>
-   *  Inputs are the logits, not probabilities.
+   * Computes softmax cross entropy between <code>logits</code> and <code>labels</code>.
    *
-   * @param <T> data type for {@code loss()} output
-   * @param features batch_size x num_classes matrix
-   * @param labels batch_size x num_classes matrix
-   *  The caller must ensure that each batch of labels represents a valid
-   *  probability distribution.
-   * @return a new instance of SoftmaxCrossEntropyWithLogits
+   *  <p>Measures the probability error in discrete classification tasks in which the classes are
+   *  mutually exclusive (each entry is in exactly one class). For example, each CIFAR-10 image is
+   *  labeled with one and only one label: an image can be a dog or a truck, but not both.
+   *
+   *  <p><b>NOTE:</b>
+   *
+   *  <p>While the classes are mutually exclusive, their probabilities need not be. All that is
+   *  required is that each row of <code>labels</code> is a valid probability distribution. If they
+   *  are not, the computation of the gradient will be incorrect.
+   *
+   *  <p>If using exclusive <code>labels</code> (wherein one and only one class is true at a time),
+   *  see {@link org.tensorflow.op.NnOps#sparseSoftmaxCrossEntropyWithLogits}
+   *
+   *  <p>Usage:
+   *
+   *  <pre>
+   *    Operand&lt;TFloat32&gt; logits =
+   *        tf.constant(new float[][] {{4.0F, 2.0F, 1.0F}, {0.0F, 5.0F, 1.0F}} );
+   *    Operand&lt;TFloat32&gt; labels =
+   *        tf.constant(new float[][] {{1.0F, 0.0F, 0.0F}, {0.0F, 0.8F, 0.2F}} );
+   *    Operand&lt;TFloat32&gt; output =
+   *        tf.nn.softmaxCrossEntropyWithLogits(labels, logits, -1);
+   *    // output Shape = [2]
+   *    // dataType = FLOAT (1)
+   *    // values { 0.169846, 0.824745 }
+   *  </pre>
+   *
+   *  <p>Backpropagation will happen into both <code>logits</code> and <code>labels</code>. To
+   *  disallow backpropagation into <code>labels</code>, pass label tensors through <code>
+   *  tf.stopGradient</code> before feeding it to this function.
+   *
+   * @param scope current scope
+   * @param labels Each vector along the class dimension should hold a valid probability
+   *      distribution e.g. for the case in which labels are of shape <code>[batch_size, num_classes]
+   *      </code>, each row of <code>labels[i]</code> must be a valid probability distribution.
+   * @param logits Per-label activations, typically a linear output. These activation energies are
+   *      interpreted as unnormalized log probabilities.
+   * @param axis The class dimension. -1 is the last dimension.
+   * @param <T> the number type of the operands
+   * @return the softmax cross entropy loss. Its type is the same as <code>logits</code> and its
+   *      shape is the same as <code>labels</code> except that it does not have the last dimension of
+   *      <code>labels</code>.
    */
-  public <T extends TNumber> SoftmaxCrossEntropyWithLogits<T> softmaxCrossEntropyWithLogits(
-      Operand<T> features, Operand<T> labels) {
-    return SoftmaxCrossEntropyWithLogits.create(scope, features, labels);
+  public <T extends TNumber, U extends TNumber> Operand<T> softmaxCrossEntropyWithLogits(
+      Operand<U> labels, Operand<T> logits, int axis) {
+    return SoftmaxCrossEntropyWithLogits.softmaxCrossEntropyWithLogits(scope, labels, logits, axis);
   }
 
   /**
@@ -1988,24 +2075,51 @@ public <T extends TType> SpaceToDepth<T> spaceToDepth(Operand<T> input, Long blo
   }
 
   /**
-   * Computes softmax cross entropy cost and gradients to backpropagate.
-   *  <p>
-   *  Unlike `SoftmaxCrossEntropyWithLogits`, this operation does not accept
-   *  a matrix of label probabilities, but rather a single label per row
-   *  of features.  This label is considered to have probability 1.0 for the
-   *  given row.
-   *  <p>
-   *  Inputs are the logits, not probabilities.
-   *
-   * @param <T> data type for {@code loss()} output
-   * @param features batch_size x num_classes matrix
-   * @param labels batch_size vector with values in [0, num_classes).
-   *  This is the label for the given minibatch entry.
-   * @return a new instance of SparseSoftmaxCrossEntropyWithLogits
-   */
-  public <T extends TNumber, U extends TNumber> SparseSoftmaxCrossEntropyWithLogits<T> sparseSoftmaxCrossEntropyWithLogits(
-      Operand<T> features, Operand<U> labels) {
-    return SparseSoftmaxCrossEntropyWithLogits.create(scope, features, labels);
+   * Computes sparse softmax cross entropy between <code>logits</code> and <code>labels</code>.
+   *
+   *  <p>Measures the probability error in discrete classification tasks in which the classes are
+   *  mutually exclusive (each entry is in exactly one class). For example, each CIFAR-10 image is
+   *  labeled with one and only one label: an image can be a dog or a truck, but not both.
+   *
+   *  <p><b>NOTE:</b>
+   *
+   *  <p>For this operation, the probability of a given label is considered exclusive. That is, soft
+   *  classes are not allowed, and the <code>labels</code> vector must provide a single specific
+   *  index for the true class for each row of <code>logits</code> (each minibatch entry). For soft
+   *  softmax classification with a probability distribution for each entry, {@link
+   *  org.tensorflow.op.NnOps#softmaxCrossEntropyWithLogits}.
+   *
+   *  <p><b>WARNING:</b>
+   *
+   *  <p>This op expects unscaled logits, since it performs a <code>softmax</code> on <code>logits
+   *  </code> internally for efficiency. Do not call this op with the output of <code>softmax</code>,
+   *  as it will produce incorrect results.
+   *
+   *  <p>A common use case is to have logits of shape <code>[batchSize, numClasses]</code> and have
+   *  labels of shape <code>[batchSize]</code>, but higher dimensions are supported, in which case
+   *  the <code>dim</code>-th dimension is assumed to be of size <code>numClasses</code>. <code>
+   *  logits</code> must have the <cod>dataType</cod> of <code>TFloat16</code>, <code>TFloat32</code>
+   *  , or <code>TFloat64</code>, and <code>labels</code> must have the dtype of <code>TInt32</code>
+   *  or <code>TInt64</code>.
+   *
+   * @param scope current scope
+   * @param labels <code>Tensor</code> of shape <code>[d_0, d_1, ..., d_{r-1}]</code> (where <code>r
+   *      </code> is rank of <code>labels</code> and result) and the dataType is <code>TInt32</code>
+   *      or <code>TInt64</code>. Each entry in <code>labels</code> must be an index in <code>[0,
+   *      numClasses)</code>. Other values will raise an exception when this op is run on CPU, and
+   *      return <code>NaN</code> for corresponding loss and gradient rows on GPU.
+   * @param logits Per-label activations (typically a linear output) of shape <code>[d_0, d_1, ...,
+   *      d_{r-1}, numClasses]</code> and dataType of <code>TFloat16</code>, <code>TFloat32</code>,
+   *      or <code>TFloat64</code>. These activation energies are interpreted as unnormalized log
+   *      probabilities.
+   * @return A <code>Tensor</code> of the same shape as <code>labels</code> and of the same type as
+   *      <code>logits</code> with the softmax cross entropy loss.
+   * @throws IllegalArgumentException If logits are scalars (need to have rank >= 1) or if the rank
+   *      of the labels is not equal to the rank of the logits minus one.
+   */
+  public <T extends TNumber, U extends TNumber> Operand sparseSoftmaxCrossEntropyWithLogits(
+      Operand<T> labels, Operand<U> logits) {
+    return SparseSoftmaxCrossEntropyWithLogits.sparseSoftmaxCrossEntropyWithLogits(scope, labels, logits);
   }
 
   /**

tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/types/TStringTest.java

@@ -39,13 +39,10 @@ public void createScalar() {
 
     @Test
     public void createrScalarLongerThan127() {
-        Tensor<TString> tensor = TString.scalarOf("Long String 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 !");
+        TString tensor = TString.scalarOf("Long String 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 !");
         assertNotNull(tensor);
-
-        TString data = tensor.data();
-        assertNotNull(data);
-        assertEquals(Shape.scalar(), data.shape());
-        assertEquals("Long String 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 !", data.getObject());
+        assertEquals(Shape.scalar(), tensor.shape());
+        assertEquals("Long String 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 !", tensor.getObject());
     }
 
 

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/ELU.java

@@ -14,7 +14,6 @@
 =======================================================================*/
 package org.tensorflow.framework.activations;
 
-import org.tensorflow.DataType;
 import org.tensorflow.Operand;
 import org.tensorflow.op.Ops;
 import org.tensorflow.types.TBool;
@@ -89,7 +88,7 @@ public Operand<T> call(Operand<T> input) {
     Operand<T> result = tf.nn.elu(input);
     if (alpha == 1.0) return result;
     else {
-      DataType<T> dataType = input.asOutput().dataType();
+      Class<T> dataType = input.asOutput().type();
       Operand<T> y = tf.math.mul(result, tf.dtypes.cast(tf.constant(alpha), dataType));
       Operand<TBool> cond = tf.math.greater(result, tf.dtypes.cast(tf.constant(0), dataType));
       return tf.select(cond, result, y);

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/HardSigmoid.java

@@ -14,7 +14,6 @@
 =======================================================================*/
 package org.tensorflow.framework.activations;
 
-import org.tensorflow.DataType;
 import org.tensorflow.Operand;
 import org.tensorflow.op.Ops;
 import org.tensorflow.types.family.TFloating;
@@ -63,7 +62,7 @@ public HardSigmoid(Ops tf) {
    */
   @Override
   public Operand<T> call(Operand<T> input) {
-    DataType<T> dataType = input.asOutput().dataType();
+    Class<T> dataType = input.asOutput().type();
     Operand<T> point2 = tf.dtypes.cast(tf.constant(0.2), dataType);
     Operand<T> point5 = tf.dtypes.cast(tf.constant(0.5), dataType);
 

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/ReLU.java

@@ -14,7 +14,6 @@
 =======================================================================*/
 package org.tensorflow.framework.activations;
 
-import org.tensorflow.DataType;
 import org.tensorflow.Operand;
 import org.tensorflow.op.Ops;
 import org.tensorflow.op.math.Greater;
@@ -99,7 +98,7 @@ public ReLU(Ops tf, float alpha, float maxValue, float threshold) {
   @Override
   public Operand<T> call(Operand<T> input) {
 
-    DataType<T> dataType = input.asOutput().dataType();
+    Class<T> dataType = input.asOutput().type();
 
     boolean clipMax = !Float.isNaN(maxValue);
     Operand<T> negativePart = null;

tensorflow-framework/src/main/java/org/tensorflow/framework/losses/BinaryCrossentropy.java

@@ -217,8 +217,8 @@ public <T extends TNumber, U extends TNumber> Operand<T> call(
               getTF(),
               "predictions range check [0-1]",
               predictions,
-              cast(getTF(), getTF().constant(0), predictions.asOutput().dataType()),
-              cast(getTF(), getTF().constant(1), predictions.asOutput().dataType()));
+              cast(getTF(), getTF().constant(0), predictions.asOutput().type()),
+              cast(getTF(), getTF().constant(1), predictions.asOutput().type()));
 
     } else {
       lPredictions = predictions;

tensorflow-framework/src/main/java/org/tensorflow/framework/losses/CategoricalCrossentropy.java

@@ -256,8 +256,8 @@ public <T extends TNumber, U extends TNumber> Operand<T> call(
               getTF(),
               "predictions range check [0-1]",
               predictions,
-              cast(getTF(), getTF().constant(0), predictions.asOutput().dataType()),
-              cast(getTF(), getTF().constant(1), predictions.asOutput().dataType()));
+              cast(getTF(), getTF().constant(0), predictions.asOutput().type()),
+              cast(getTF(), getTF().constant(1), predictions.asOutput().type()));
 
     } else {
       lPredictions = predictions;

tensorflow-framework/src/main/java/org/tensorflow/framework/losses/Hinge.java

@@ -124,15 +124,15 @@ public Hinge(Ops tf, String name, Reduction reduction) {
   public <T extends TNumber, U extends TNumber> Operand<T> call(
       Operand<U> labels, Operand<T> predictions, Operand<T> sampleWeights) {
     @SuppressWarnings("unchecked")
-    Operand<T> tLabels = predictions.asOutput().dataType() == labels.asOutput().dataType() ?
+    Operand<T> tLabels = predictions.asOutput().type() == labels.asOutput().type() ?
             (Operand<T>)labels :
-            cast(tf,  labels, predictions.asOutput().dataType());
+            cast(tf,  labels, predictions.asOutput().type());
     tLabels = LossesHelper.valueCheck(
             getTF(),
             "labels value check [-1, 0, 1]",
             tLabels,
             cast(getTF(), getTF().constant(new int[] { -1, 0, 1}),
-                    predictions.asOutput().dataType()));
+                    predictions.asOutput().type()));
 
     Operand<T> losses = Losses.hinge(getTF(), tLabels, predictions);
     return LossesHelper.computeWeightedLoss(getTF(), losses, getReduction(), sampleWeights);