From 11d6dc4a4eb97d78398490e28953305ab7a0c756 Mon Sep 17 00:00:00 2001 From: Ningxin Hu Date: Tue, 23 Feb 2021 16:10:32 +0800 Subject: [PATCH 1/6] Add the spec and examples of Compilation interface --- index.bs | 377 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 377 insertions(+) diff --git a/index.bs b/index.bs index 51f28275..16a9d184 100644 --- a/index.bs +++ b/index.bs @@ -30,6 +30,142 @@ urlPrefix: https://gpuweb.github.io/gpuweb/; spec: WEBGPU text: GPUTexture; url: texture-interface +
+spec:html;
+    type:interface; text:Navigator
+spec:webidl;
+    type:dfn; text:record
+
+ + + Introduction {#intro} ===================== @@ -1396,6 +1532,218 @@ interface MLGraph { }; +{{Compilation}} has the following internal slots: + +
+ : \[[inputOperands]] of type [=record=]<{{DOMString}}, {{OperandDescriptor}}>. + :: + Maps the name of an input {{Operand}} to its {{OperandDescriptor}} for all input {{Operand}}s of the {{Model}} that this {{Compilation}} is compiled from. + + : \[[outputOperands]] of type [=sequence=]<{{DOMString}}>. + :: + Contains the names of all output {{Operand}}s of the {{Model}} that this {{Compilation}} is compiled from. + + : \[[implementation]] + :: + Underlying compilation implemenation provided by the User Agent. +
+ +
+ : compute(inputs, outputs) + :: + Schedules an asynchronous computation of the {{Compilation}} given {{NamedInputs}} and optional {{NamedOutputs}}. The returned {{Promise}} resolves when the computaton results in {{NamedOutputs}} are ready to be consumed. + +
+ **Called on:** {{Compilation}} |this|. + + **Arguments:** + 
+                |inputs|: The given computation inputs of type {{NamedInputs}}. Specifies the input buffers and optional dimensions.
+                |outputs|: The optional given outputs of type {{NamedOutputs}}. Specifies the output names and pre-allocated buffers. Default to be an empty [=record=].
+
+ + **Returns:** {{Promise}}<{{NamedOutputs}}> + + 1. Let |promise| be [=a new promise=]. + + 1. If any of the following requirements are unmet, [=reject=] |promise| with a {{TypeError}} and stop. + +
+ 1. For each |key| -> |value| of |inputs|: + 1. |this|.{{Compilation/[[inputOperands]]}}[|key|] must exist. + 1. Let |inputOperand| be |this|.{{Compilation/[[inputOperands]]}}[|key|]. + 1. The |value|.{{Input/buffer}} of type {{ArrayBufferView}} must be compatible to |inputOperand|.{{OperandDescriptor/type}} according to [this table](#operandtype-arraybufferview-compatibility). + 1. If |value|.{{Input/dimensions}} was given, then: + 1. The size of |value|.{{Input/dimensions}} must be the same as the size of |inputOperand|.{{OperandDescriptor/dimensions}}. + 1. Let |i| be 0. + 1. While true: + 1. Let |dimension| be |value|.{{Input/dimensions}}[|i|]. + 1. The value of |dimension| must be greater than 0. + 1. If the value of |inputOperand|.{{OperandDescriptor/dimensions}}[|i|] is greater than 0, then: + 1. The value of |dimension| must be the same as the value of |inputOperand|.{{OperandDescriptor/dimensions}}[|i|]. + 1. Set |i| to |i| + 1. + 1. If |i| equals to the size of |value|.{{Input/dimensions}}, then break. + 1. Otherwise: + 1. For each |dimension| of |inputOperand|.{{OperandDescriptor/dimensions}}: + 1. The value of |dimension| must be greater than 0. + + 1. If |outputs| was not an empty [=record=], then: + 1. For each |key| -> |value| of |outputs|: + 1. |this|.{{Compilation/[[outputOperands]]}}[|key|] must exist. + 1. If |value|.{{Output/buffer}} was given, then: + 1. Let |outputOperand| be |this|.{{Compilation/[[outputOperands]]}}[|key|]. + 1. The |value|.{{Output/buffer}} of type {{ArrayBufferView}} must be compatible to |outputOperand|.{{OperandDescriptor/type}} according to [this table](#operandtype-arraybufferview-compatibility). +
+ + 1. Let |requiredOutputNames| be a new [=ordered set=]<{{DOMString}}>. + 1. If |outputs| was not an empty [=record=], then: + 1. For each |key| -> |value| of |outputs|: + 1. Append |key| to |requiredOutputNames|. + 1. Otherwise: + 1. For each |key| -> |value| of |this|.{{Compilation/[[outputOperands]]}}: + 1. Append |key| to |requiredOutputNames|. + + 1. Let |copiedInputs| be a new {{NamedInputs}}. + 1. For each |key| -> |value| of |inputs|: + 1. Let |input| be a new {{Input}}. + 1. Let |input|.{{Input/buffer}} be a new {{ArrayBufferView}} that has the same kind as |value|.{{Input/buffer}}'s. + 1. Copy the bytes held by |value|.{{Input/buffer}} to |input|.{{Input/buffer}}. + 1. Let |input|.{{Input/dimensions}} be a new [=sequence=]<{{long}}>. + 1. Copy the values heold by |value|.{{Input/dimensions}} to |input|.{{Input/dimensions}}. + 1. Set |copiedInputs|[key] to |input|. + + 1. Let |results| be a new {{NamedOutputs}}. + 1. Let |remainingOutputNames| be a new [=ordered set=]<{{DOMString}}>. + 1. For each |outputName| of |requiredOutputNames|: + 1. Append |outputName| to |remainingOutputNames|. + 1. For each |outputName| of |requiredOutputNames|: + 1. Issue an asynchronous computation of |this|.{{Compilation/[[implementation]]}} for output whose name is |outputName| with given |copiedInputs|. + 1. When the asynchronous computation is completed: + 1. If there is an error returned by |this|.{{Compilation/[[implementation]]}}, then: + 1. [=reject=] |promise| with an {{OperationError}} and stop. + 1. Otherwise: + 1. Let |outputDemisions| be a new [=sequence=]<{{long}}> that contains the dimensions returned by |this|.{{Compilation/[[implementation]]}} for |outputName|. + 1. Let |outputBuffer| be a new {{ArrayBufferView}} that holds the bytes returned by |this|.{{Compilation/[[implementation]]}} for |outputName|. + 1. Set |results|[|outputName|].{{Output/dimensions}} to |outputDemisions|. + 1. If |outputs|[|outputName|].{{Output/buffer}} was given, then: + 1. If the kind of |outputs|[|outputName|].{{Output/buffer}} is not the same as |outputBuffer|, then: + 1. [=reject=] |promise| with a {{TypeError}} and stop. + 1. If the length of |outputs|[|outputName|].{{Output/buffer}} is not the same as |outputBuffer|, then: + 1. [=reject=] |promise| with a {{TypeError}} and stop. + 1. Copy the bytes held by |outputBuffer| to |outputs|[|outputName|].{{Output/buffer}}. + 1. Otherwise: + 1. Let |buffer| be a new {{ArrayBufferView}} that has the same kind and length as |outputBuffer|. + 1. Copy the bytes held by |outputBuffer| to |buffer|. + 1. Set |results|[|outputName|].{{Output/buffer}} to |buffer|. + 1. Remove |outputName| from |remainingOutputNames|. + 1. If |remainingOutputNames| is empty, then: + 1. Resolve |promise| with |results|. + 1. Return |promise|. +
+
+ +### Examples ### {#compilation-examples} + +
+The following code showcases the computation with dynamic input dimensions. +
+const nn = navigator.ml.getNeuralNetworkContext();
+
+// Create a model with dynamic shaped inputs.
+const builder = nn.createModelBuilder();
+const descA = {type: 'float32', dimensions: [-1, 4]};
+const a = builder.input('a', descA);
+const descB = {type: 'float32', dimensions: [4, -1]};
+const b = builder.input('b', descB);
+const c = builder.matmul(a, b);
+const model = builder.createModel({c});
+
+const compiledModel = await model.compile();
+
+async function compute(shapeA, shapeB) {
+  const bufferA = new Float32Array(sizeOfShape(shapeA)).fill(0.5);
+  const bufferB = new Float32Array(sizeOfShape(shapeB)).fill(0.5);
+
+  // Specify the shape of inputs when computing.
+  const inputs = {
+    'a': {buffer: bufferA, dimensions: shapeA},
+    'b': {buffer: bufferB, dimensions: shapeB},
+  };
+  const outputs = await compiledModel.compute(inputs);
+  console.log(`shape: [${outputs.c.dimensions}], values: ${outputs.c.buffer}`);
+}
+
+await compute([3, 4], [4, 3]);
+await compute([4, 4], [4, 4]);
+await compute([5, 4], [4, 5]);
+
+
+ +
+The following code showcases the computation with pre-allocated output buffers. +
+const nn = navigator.ml.getNeuralNetworkContext();
+
+// The following code multiplies matrix a [3, 4] with matrix b [4, 3]
+// into matrix c [3, 3].
+const builder = nn.createModelBuilder();
+const descA = {type: 'float32', dimensions: [3, 4]};
+const a = builder.input('a', descA);
+const descB = {type: 'float32', dimensions: [4, 3]};
+const bufferB = new Float32Array(sizeOfShape(descB.dimensions)).fill(0.5);
+const b = builder.constant(descB, bufferB);
+const c = builder.matmul(a, b);
+const model = builder.createModel({c});
+
+const compiledModel = await model.compile();
+const bufferA = new Float32Array(sizeOfShape(descA.dimensions)).fill(0.5);
+const inputs = {'a': {buffer: bufferA}};
+// Pre-allocate output buffer for c.
+const outputs = {'c': {buffer: new Float32Array(sizeOfShape([3, 3]))}};
+await compiledModel.compute(inputs, outputs);
+console.log(`values: ${outputs.c.buffer}`);
+
+
+ +
+The following code showcases the computation with optional outputs. +
+const nn = navigator.ml.getNeuralNetworkContext();
+
+// Build a model with two outputs.
+const builder = nn.createModelBuilder();
+const descA = {type: 'float32', dimensions: [3, 4]};
+const a = builder.input('a', descA);
+const descB = {type: 'float32', dimensions: [4, 3]};
+const bufferB = new Float32Array(sizeOfShape(descB.dimensions)).fill(0.5);
+const b = builder.constant(descB, bufferB);
+const descC = {type: 'float32', dimensions: [3, 3]};
+const bufferC = new Float32Array(sizeOfShape(descC.dimensions)).fill(1);
+const c = builder.constant(descC, bufferC);
+const d = builder.matmul(a, b);
+const e = builder.add(d, c);
+const model = builder.createModel({d, e});
+
+const compiledModel = await model.compile();
+const bufferA = new Float32Array(sizeOfShape(descA.dimensions)).fill(0.5);
+const inputs = {'a': {buffer: bufferA}};
+
+// Compute both d and e.
+let outputs = await compiledModel.compute(inputs);
+console.log(`outputs include ${Object.keys(outputs)}`);
+
+// Compute d.
+outputs = await compiledModel.compute(inputs, {d});
+console.log(`outputs include ${Object.keys(outputs)}`);
+console.log(`shape: [${outputs.d.dimensions}], values: ${outputs.d.buffer}`);
+
+// Compute e.
+outputs = await compiledModel.compute(inputs, {e});
+console.log(`outputs include ${Object.keys(outputs)}`);
+console.log(`shape: [${outputs.e.dimensions}], values: ${outputs.e.buffer}`);
+
+
+ Examples {#examples} ===================== @@ -1482,6 +1830,35 @@ console.log('Output value: ' + outputs.output.data); +# Appendices # {#appendices} + +## {{OperandType}} and {{ArrayBufferView}} compatibility ## {#operandtype-arraybufferview-compatibility} + + + + + + + + + + +
{{OperandType}} + {{ArrayBufferView}} +
{{OperandType/float32}} + {{Float32Array}} +
{{OperandType/int32}} + {{Int32Array}} +
{{OperandType/uint32}} + {{Uint32Array}} +
{{OperandType/int8}} + {{Int8Array}} +
{{OperandType/uint8}} + {{Uint8Array}} +
+ +Issue(webmachinelearning/webnn#127): clarify the usage of {{ArrayBufferView}} for {{OperandType/float16}}. +

Acknowledgements

This specification follows the concepts of the Android Neural Networks API C From 818786439c0ce9c1f4015695a1ade2ae4936d30f Mon Sep 17 00:00:00 2001 From: Ningxin Hu Date: Wed, 24 Feb 2021 08:12:27 +0800 Subject: [PATCH 2/6] Add live example links --- index.bs | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/index.bs b/index.bs index 16a9d184..c8345701 100644 --- a/index.bs +++ b/index.bs @@ -1566,7 +1566,7 @@ interface MLGraph { 1. Let |promise| be [=a new promise=]. - 1. If any of the following requirements are unmet, [=reject=] |promise| with a {{TypeError}} and stop. + 1. If any of the following requirements are unmet, then [=reject=] |promise| with a {{TypeError}} and stop.
1. For each |key| -> |value| of |inputs|: @@ -1622,22 +1622,19 @@ interface MLGraph { 1. If there is an error returned by |this|.{{Compilation/[[implementation]]}}, then: 1. [=reject=] |promise| with an {{OperationError}} and stop. 1. Otherwise: - 1. Let |outputDemisions| be a new [=sequence=]<{{long}}> that contains the dimensions returned by |this|.{{Compilation/[[implementation]]}} for |outputName|. - 1. Let |outputBuffer| be a new {{ArrayBufferView}} that holds the bytes returned by |this|.{{Compilation/[[implementation]]}} for |outputName|. - 1. Set |results|[|outputName|].{{Output/dimensions}} to |outputDemisions|. - 1. If |outputs|[|outputName|].{{Output/buffer}} was given, then: - 1. If the kind of |outputs|[|outputName|].{{Output/buffer}} is not the same as |outputBuffer|, then: - 1. [=reject=] |promise| with a {{TypeError}} and stop. - 1. If the length of |outputs|[|outputName|].{{Output/buffer}} is not the same as |outputBuffer|, then: - 1. [=reject=] |promise| with a {{TypeError}} and stop. - 1. Copy the bytes held by |outputBuffer| to |outputs|[|outputName|].{{Output/buffer}}. - 1. Otherwise: - 1. Let |buffer| be a new {{ArrayBufferView}} that has the same kind and length as |outputBuffer|. - 1. Copy the bytes held by |outputBuffer| to |buffer|. - 1. Set |results|[|outputName|].{{Output/buffer}} to |buffer|. - 1. Remove |outputName| from |remainingOutputNames|. - 1. If |remainingOutputNames| is empty, then: - 1. Resolve |promise| with |results|. + 1. Let |outputDemisions| be a new [=sequence=]<{{long}}> that contains the dimensions returned by |this|.{{Compilation/[[implementation]]}} for |outputName|. + 1. Let |outputBuffer| be a new {{ArrayBufferView}} that holds the bytes returned by |this|.{{Compilation/[[implementation]]}} for |outputName|. + 1. Set |results|[|outputName|].{{Output/dimensions}} to |outputDemisions|. + 1. If |outputs|[|outputName|].{{Output/buffer}} was given, then: + 1. If the kind of |outputs|[|outputName|].{{Output/buffer}} is not the same as |outputBuffer|, then [=reject=] |promise| with a {{TypeError}} and stop. + 1. If the length of |outputs|[|outputName|].{{Output/buffer}} is not the same as |outputBuffer|, then [=reject=] |promise| with a {{TypeError}} and stop. + 1. Copy the bytes held by |outputBuffer| to |outputs|[|outputName|].{{Output/buffer}}. + 1. Otherwise: + 1. Let |buffer| be a new {{ArrayBufferView}} that has the same kind and length as |outputBuffer|. + 1. Copy the bytes held by |outputBuffer| to |buffer|. + 1. Set |results|[|outputName|].{{Output/buffer}} to |buffer|. + 1. Remove |outputName| from |remainingOutputNames|. + 1. If |remainingOutputNames| is empty, then resolve |promise| with |results|. 1. Return |promise|.
@@ -1645,7 +1642,7 @@ interface MLGraph { ### Examples ### {#compilation-examples}
-The following code showcases the computation with dynamic input dimensions. +The following code showcases the computation with dynamic input dimensions. Check out [a live version](https://webmachinelearning.github.io/webnn-samples/code/?example=dynamic_shape.js) of this example in WebNN Code Editor. 
 const nn = navigator.ml.getNeuralNetworkContext();
 
@@ -1680,7 +1677,7 @@ await compute([5, 4], [4, 5]);
-The following code showcases the computation with pre-allocated output buffers. +The following code showcases the computation with pre-allocated output buffers. Check out [a live version](https://webmachinelearning.github.io/webnn-samples/code/?example=preallocated_outputs.js) of this example in WebNN Code Editor. 
 const nn = navigator.ml.getNeuralNetworkContext();
 
@@ -1706,7 +1703,7 @@ console.log(`values: ${outputs.c.buffer}`);
-The following code showcases the computation with optional outputs. +The following code showcases the computation with optional outputs. Check out [a live version](https://webmachinelearning.github.io/webnn-samples/code/?example=optional_outputs.js) of this example in WebNN Code Editor. 
 const nn = navigator.ml.getNeuralNetworkContext();
 

From f11aaba7fcdc196d904b2affdb67e6ffc9757f53 Mon Sep 17 00:00:00 2001
From: Ningxin Hu 
Date: Wed, 24 Feb 2021 10:19:36 +0800
Subject: [PATCH 3/6] Add programing model and timelines

---
 index.bs | 190 ++++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 145 insertions(+), 45 deletions(-)

diff --git a/index.bs b/index.bs
index c8345701..9426093e 100644
--- a/index.bs
+++ b/index.bs
@@ -35,6 +35,7 @@ spec:html;
     type:interface; text:Navigator
 spec:webidl;
     type:dfn; text:record
+    type:dfn; text:resolve