From 6e137e40a745e901b15f5374df8009f40f1d3401 Mon Sep 17 00:00:00 2001 From: Chai Chaoweeraprasit Date: Mon, 12 Apr 2021 22:27:45 -0700 Subject: [PATCH 1/6] Support device selection. Explain device resource expectations and constraints. Introduce MLTensor that allows asynchronous readback. --- explainer.md | 19 ++--- index.bs | 208 ++++++++++++++++++++++++++++++++++----------------- 2 files changed, 148 insertions(+), 79 deletions(-) diff --git a/explainer.md b/explainer.md index abd7794b..8d5d4d00 100644 --- a/explainer.md +++ b/explainer.md @@ -44,12 +44,13 @@ const graph = await builder.build({'C': C}); // 3. Bind inputs to the graph and execute for the result. const bufferA = new Float32Array(4).fill(1.0); const bufferB = new Float32Array(4).fill(0.8); -const inputs = {'A': {data: bufferA}, 'B': {data: bufferB}}; -const outputs = await graph.compute(inputs); +const inputs = {'A': new MLTensor(bufferA), 'B': new MLTensor(bufferB)}; +const outputs = graph.compute(inputs); +const data = await outputs.C.data(); // The computed result of [[1, 1], [1, 1]] is in the buffer associated with // the output operand. -console.log('Output shape: ' + outputs.C.dimensions); -console.log('Output value: ' + outputs.C.data); +console.log('Output shape: ' + outputs.C.dimensions()); +console.log('Output value: ' + data); ``` Check it out in [WebNN Code Editor](https://webmachinelearning.github.io/webnn-samples/code/?example=mul_add.js). @@ -145,13 +146,13 @@ export class NSNet2 { this.graph = await this.builder.build({output, gru94, gru157}); } - async compute(inputBuffer, initialState92Buffer, initialState155Buffer) { + compute(inputBuffer, initialState92Buffer, initialState155Buffer) { const inputs = { - input: {data: inputBuffer}, - initialState92: {data: initialState92Buffer}, - initialState155: {data: initialState155Buffer}, + input: new MLTensor(inputBuffer), + initialState92: new MLTensor(initialState92Buffer), + initialState155: new MLTensor(initialState155Buffer) }; - return await this.graph.compute(inputs); + return this.graph.compute(inputs); } } ``` diff --git a/index.bs b/index.bs index 53acd0b6..c43a4f19 100644 --- a/index.bs +++ b/index.bs @@ -201,6 +201,20 @@ thead.stickyheader th, th.stickyheader { background: var(--stickyheader-background); } +/* + * Generic table format. + */ +th { + text-align: left; +} + +th, td { + border-bottom: 1px solid black; + border-collapse: collapse; + padding-left: 5px; + padding-right: 5px; +} + /* * Darkmode colors */ @@ -222,7 +236,6 @@ thead.stickyheader th, th.stickyheader { --tint-purple: rgba(255, 0, 255, 22%); } } - Introduction {#intro} @@ -429,10 +442,38 @@ They are represented by callbacks and promises in JavaScript.
{{MLGraph/compute()|MLGraph.compute()}}: - 1. User issues a compute request by calling {{MLGraph/compute()|MLGraph.compute()}} on the [=Content timeline=] and gets a promise in return. - 2. User agent processes the compute request on the [=Device timeline=] by calling the OS ML API. - 3. After the ML device operating on [=Queue timeline=] is done, the user agent makes the results ready to be consumed by user and [=resolves=] the promise. + 1. User issues a compute request by calling {{MLGraph/compute()|MLGraph.compute()}} on the [=Content timeline=]. + 2. user agent processes the compute request on the [=Device timeline=] by calling the OS ML API. + 3. After the ML device operating on [=Queue timeline=] is done, the output is returned. + 4. User issues a data download request by calling {{MLTensor/data()|MLTensor/data()}} on the [=Content timeline=] and gets a promise in return. + 5. user agent processes the download request on the [=Device timeline=] by calling the OS API. + 6. After the data download on [=Queue timeline=] is done, the data is available to the user. + 7. user agent makes the results ready to be consumed by the user and [=resolves=] the promise. + +
+ +## Device Selection ## {#programming-model-device-selection} + +An {{MLContext}} interface represents a global state of neural network execution. An important context state is the underlying execution device that manages the resources and facilitates the compilation and eventual execution of the neural network graph. {{MLContext}} may be created from a specific GPU device such as {{GPUDevice}} or {{WebGLRenderingContext}} that is already in use by the application, in which case the corresponding {{GPUBuffer}} or {{WebGLBuffer}} resources used as graph constants, as well as the {{GPUTexture}} and {{WebGLTexture}} as graph inputs must also be created from the same device. In a multi-adapter configuration, the device used for {{MLContext}} must be created from the same adapter as the device used to allocate and manage the resources referenced in the graph. If a graph constant or an input is an {{ArrayBufferView}} in the system memory, it is automatically uploaded from the system memory to the GPU memory, and vice versa downloaded to an {{ArrayBufferView}} for a graph output. + +When an {{MLContext}} is created with {{MLContextOptions}}, the user agent selects and creates the underlying execution device by taking into account the application's preference as defined in the {{MLDevicePreference}} option. +- The *"graphics"* device provides the broadest range of achievable performance across hardware platforms from consumer devices to professional workstations. +- The *"compute"* device takes advantage of dedicated compute devices for specific kinds of workload with great power efficiency over a long period of execution time. +- The *"software"* device provides the broadest reach of compute availability, but with limited scalability of execution performance on the more complex neural networks. +- When the device preference is not specified (*"default"*), the user agent selects the most suitable device to use. +The following table summarizes all the supported device types and the resource types of constants, inputs, and outputs that each of the device types supports. + +
+ +
Device TypeArrayBufferViewGPUBufferGPUTextureWebGLBufferWebGLTexture +
GPUDeviceYesYesYesNoNo +
WebGLRenderingContextYesNoNoYesYes +
defaultYesNoNoNoNo +
graphicsYesNoNoNoNo +
computeYesNoNoNoNo +
softwareYesNoNoNoNo +
API {#api} @@ -447,16 +488,29 @@ partial interface Navigator { ## ML ## {#api-ml} -typedef record MLNamedInputs; -typedef record MLNamedOutputs; +## MLGraph ## {#api-mlgraph} +The {{MLGraph}} interface represents a compiled computational graph. A compiled graph once constructed is immutable and cannot be subsequently changed. + + @@ -1657,44 +1721,43 @@ interface MLGraph { : \[[implementation]] :: - The underlying implemenation provided by the User Agent. + The underlying implemenation provided by the user agent.
: compute(inputs, outputs) :: - Issue a compute request of the {{MLGraph}} given {{MLNamedInputs}} and optional {{MLNamedOutputs}}. The returned {{Promise}} resolves when the results in {{MLNamedOutputs}} are ready to be consumed. + Issue a compute request of the {{MLGraph}} given an input {{MLNamedTensors}} and optional output {{MLNamedTensors}}. The returned {{MLNamedTensors}} are the results ready to be consumed.
**Called on:** {{MLGraph}} |this|. **Arguments:** 
-                |inputs|: a {{MLNamedInputs}}. The data and optional dimensions of inputs for the compute request.
-                |outputs|: an optional {{MLNamedOutputs}}. The names and pre-allocated resources of required outputs for the compute request. Default to be an empty [=record=] which means that the compute request is for all outputs.
+                |inputs|: a {{MLNamedTensors}}. The data and optional dimensions of inputs for the compute request.
+                |outputs|: an optional {{MLNamedTensors}}. The names and pre-allocated resources of required outputs for the compute request. Default to be an empty [=record=] which means that the compute request is for all outputs.
- **Returns:** {{Promise}}<{{MLNamedOutputs}}>. The dimensions and data of outputs returned by the compute request. + **Returns:** {{MLNamedTensors}}. The dimensions and data of outputs returned by the compute request. - 1. Let |promise| be [=a new promise=]. - 1. If any of the following requirements are unmet, then [=reject=] |promise| with a {{TypeError}} and stop. + 1. If any of the following requirements are unmet, then stop.
1. For each |key| -> |value| of |inputs|: 1. |this|.{{MLGraph/[[inputOperands]]}}[|key|] must exist. 1. Let |inputOperand| be |this|.{{MLGraph/[[inputOperands]]}}[|key|]. - 1. If |value|.{{MLInput/data}} is an {{ArrayBufferView}}, then: - 1. The kind of |value|.{{MLInput/data}} must be compatible to |inputOperand|.{{MLOperandDescriptor/type}} according to [this table](#appendices-mloperandtype-arraybufferview-compatibility). - 1. If |value|.{{MLInput/dimensions}} was given, then: - 1. The length of |value|.{{MLInput/dimensions}} must be the same as the length of |inputOperand|.{{MLOperandDescriptor/dimensions}}. + 1. If |value|.{{MLTensor/resource}} is an {{ArrayBufferView}}, then: + 1. The kind of |value|.{{MLTensor/resource}} must be compatible to |inputOperand|.{{MLOperandDescriptor/type}} according to [this table](#appendices-mloperandtype-arraybufferview-compatibility). + 1. If |value|.{{MLTensor/dimensions}} was given, then: + 1. The length of |value|.{{MLTensor/dimensions}} must be the same as the length of |inputOperand|.{{MLOperandDescriptor/dimensions}}. 1. Let |i| be 0. 1. While true: - 1. Let |dimension| be |value|.{{MLInput/dimensions}}[|i|]. + 1. Let |dimension| be |value|.{{MLTensor/dimensions}}[|i|]. 1. |dimension| must be greater than 0. 1. If |inputOperand|.{{MLOperandDescriptor/dimensions}}[|i|] is greater than 0, then |dimension| must be equal to |inputOperand|.{{MLOperandDescriptor/dimensions}}[|i|]. 1. Set |i| to |i| + 1. - 1. If |i| if equal to the length of |value|.{{MLInput/dimensions}}, then break. + 1. If |i| if equal to the length of |value|.{{MLTensor/dimensions}}, then break. 1. Else: 1. For each |dimension| of |inputOperand|.{{MLOperandDescriptor/dimensions}}: 1. The value of |dimension| must be greater than 0. @@ -1702,7 +1765,7 @@ interface MLGraph { 1. If |outputs| was not an empty [=record=], then: 1. For each |key| -> |value| of |outputs|: 1. |this|.{{MLGraph/[[outputOperands]]}}[|key|] must exist. - 1. If |value|.{{MLOutput/data}} was given, then the kind of |value|.{{MLOutput/data}} must be compatible to |this|.{{MLGraph/[[outputOperands]]}}[|key|] according to [this table](#appendices-mloperandtype-arraybufferview-compatibility). + 1. If |value|.{{MLTensor/resource}} was given, then the kind of |value|.{{MLTensor/resource}} must be compatible to |this|.{{MLGraph/[[outputOperands]]}}[|key|] according to [this table](#appendices-mloperandtype-arraybufferview-compatibility).
1. Let |requiredOutputNames| be a new [=ordered set=]<{{DOMString}}>. @@ -1713,16 +1776,16 @@ interface MLGraph { 1. For each |key| -> |value| of |this|.{{MLGraph/[[outputOperands]]}}: 1. Append |key| to |requiredOutputNames|. - 1. Let |copiedInputs| be a new {{MLNamedInputs}}. + 1. Let |copiedInputs| be a new {{MLNamedTensors}}. 1. For each |key| -> |value| of |inputs|: - 1. Let |copiedInputs| be a new {{MLInput}}. - 1. Let |copiedInputs|.{{MLInput/data}} be a new {{ArrayBufferView}} that has the same kind and length as |value|.{{MLInput/data}}'s. - 1. Set the content of |copiedInputs|.{{MLInput/data}} to the content of |value|.{{MLInput/data}}. - 1. Let |copiedInputs|.{{MLInput/dimensions}} be a new [=sequence=]<{{long}}> that has the same length of |value|.{{MLInput/dimensions}}'s. - 1. Set the content of |copiedInputs|.{{MLInput/dimensions}} to the content of |value|.{{MLInput/dimensions}}. + 1. Let |copiedInputs| be a new {{MLTensor}}. + 1. Let |copiedInputs|.{{MLTensor/resource}} be a new {{ArrayBufferView}} that has the same kind and length as |value|.{{MLTensor/resource}}'s. + 1. Set the content of |copiedInputs|.{{MLTensor/resource}} to the content of |value|.{{MLTensor/resource}}. + 1. Let |copiedInputs|.{{MLTensor/dimensions}} be a new [=sequence=]<{{long}}> that has the same length of |value|.{{MLTensor/dimensions}}'s. + 1. Set the content of |copiedInputs|.{{MLTensor/dimensions}} to the content of |value|.{{MLTensor/dimensions}}. 1. Set |copiedInputs|[key] to |copiedInputs|. - 1. Let |results| be a new {{MLNamedOutputs}}. + 1. Let |results| be a new {{MLNamedTensors}}. 1. Let |remainingOutputNames| be a new [=ordered set=]<{{DOMString}}>. 1. Set the content of |remainingOutputNames| to the content of |requiredOutputNames|. 1. Issue the following steps on the [=Device timeline=] of |this|.{{MLGraph/[[implementation]]}}: @@ -1744,18 +1807,18 @@ interface MLGraph { 1. Set |outputSize| to |outputSize| * |outputDimensions|[|i|]. 1. Set |i| to |i| + 1. 1. If |i| is equal to |outputRank|, then break. - 1. Set |results|[|outputName|].{{MLOutput/dimensions}} to |outputDemisions|. + 1. Set |results|[|outputName|].{{MLTensor/dimensions}} to |outputDemisions|. 1. If |this|.{{MLGraph/[[context]]}} is created from {{MLContextOptions}}, then: - 1. If |outputs|[|outputName|].{{MLOutput/data}} was given, then: - 1. If outputs|[|outputName|].{{MLOutput/data}} is not an {{ArrayBufferView}}, then [=reject=] |promise| with an {{TypeError}} and stop. - 1. If the kind of |outputs|[|outputName|].{{MLOutput/data}} is not compatible to output tensor according to [this table](#appendices-mloperandtype-arraybufferview-compatibility), then [=reject=] |promise| with a {{TypeError}} and stop. - 1. If the length of |outputs|[|outputName|].{{MLOutput/data}} is less than |outputSize|, then [=reject=] |promise| with a {{TypeError}} and stop. - 1. Set the content of |outputs|[|outputName|].{{MLOutput/data}} to the content of output tensor returned by |this|.{{MLGraph/[[implementation]]}}. + 1. If |outputs|[|outputName|].{{MLTensor/resource}} was given, then: + 1. If outputs|[|outputName|].{{MLTensor/resource}} is not an {{ArrayBufferView}}, then stop. + 1. If the kind of |outputs|[|outputName|].{{MLTensor/resource}} is not compatible to output tensor according to [this table](#appendices-mloperandtype-arraybufferview-compatibility), then stop. + 1. If the length of |outputs|[|outputName|].{{MLTensor/resource}} is less than |outputSize|, then stop. + 1. Set the content of |outputs|[|outputName|].{{MLTensor/resource}} to the content of output tensor returned by |this|.{{MLGraph/[[implementation]]}}. 1. Else: - 1. Let |results|[|outputName|].{{MLOutput/data}} be a new {{ArrayBufferView}} of size |outputSize| and kind that is compatible to output tensor according to [this table](#appendices-mloperandtype-arraybufferview-compatibility). - 1. Set the content of |results|[|outputName|].{{MLOutput/data}} to the content of output tensor returned by |this|.{{MLGraph/[[implementation]]}}. + 1. Let |results|[|outputName|].{{MLTensor/resource}} be a new {{ArrayBufferView}} of size |outputSize| and kind that is compatible to output tensor according to [this table](#appendices-mloperandtype-arraybufferview-compatibility). + 1. Set the content of |results|[|outputName|].{{MLTensor/resource}} to the content of output tensor returned by |this|.{{MLGraph/[[implementation]]}}. 1. Remove |outputName| from |remainingOutputNames|. - 1. If |remainingOutputNames| is empty, then resolve |promise| with |results| and stop. + 1. If |remainingOutputNames| is empty, then stop.
@@ -1787,11 +1850,12 @@ async function compute(shapeA, shapeB) { // Specify the shape of inputs when computing. const inputs = { - 'a': {data: bufferA, dimensions: shapeA}, - 'b': {data: bufferB, dimensions: shapeB}, + 'a': new MLTensor(bufferA, shapeA), + 'b': new MLTensor(bufferB, shapeB) }; - const outputs = await graph.compute(inputs); - console.log(`shape: [${outputs.c.dimensions}], values: ${outputs.c.data}`); + const outputs = graph.compute(inputs); + const data = await outputs.c.data(); + console.log(`shape: [${outputs.c.dimensions()}], values: ${data}`); } await compute([3, 4], [4, 3]); @@ -1817,11 +1881,12 @@ const c = builder.matmul(a, b); const graph = await builder.build({c}); const bufferA = new Float32Array(sizeOfShape(descA.dimensions)).fill(0.5); -const inputs = {'a': {data: bufferA}}; +const inputs = {'a': new MLTensor(bufferA)}; // Pre-allocate output buffer for c. -const outputs = {'c': {data: new Float32Array(sizeOfShape([3, 3]))}}; -await graph.compute(inputs, outputs); -console.log(`values: ${outputs.c.data}`); +const outputs = {'c': new MLTensor(new Float32Array(sizeOfShape([3, 3])))}; +graph.compute(inputs, outputs); +const data = await outputs.c.data(); +console.log(`values: ${data}`); @@ -1845,21 +1910,23 @@ const e = builder.add(d, c); const graph = await builder.build({d, e}); const bufferA = new Float32Array(sizeOfShape(descA.dimensions)).fill(0.5); -const inputs = {'a': {data: bufferA}}; +const inputs = {'a': new MLTensor(bufferA)}; // Compute both d and e. -let outputs = await graph.compute(inputs); +let outputs = graph.compute(inputs); console.log(`outputs include ${Object.keys(outputs)}`); // Compute d. -outputs = await graph.compute(inputs, {d}); +outputs = graph.compute(inputs, {d}); +let data = await outputs.d.data(); console.log(`outputs include ${Object.keys(outputs)}`); -console.log(`shape: [${outputs.d.dimensions}], values: ${outputs.d.data}`); +console.log(`shape: [${outputs.d.dimensions()}], values: ${data}`); // Compute e. -outputs = await graph.compute(inputs, {e}); +outputs = graph.compute(inputs, {e}); +data = await outputs.e.data(); console.log(`outputs include ${Object.keys(outputs)}`); -console.log(`shape: [${outputs.e.dimensions}], values: ${outputs.e.data}`); +console.log(`shape: [${outputs.e.dimensions()}], values: ${data}`); @@ -1934,17 +2001,18 @@ The following code executes the compiled graph. const inputBuffer1 = new Float32Array(TENSOR_SIZE).fill(1); const inputBuffer2 = new Float32Array(TENSOR_SIZE).fill(1); -// Asynchronously execute the compiled graph with the specified inputs. +// Execute the compiled graph with the specified inputs and asynchronously download the result. const inputs = { - 'input1': {data: inputBuffer1}, - 'input2': {data: inputBuffer2}, + 'input1': new MLTensor(inputBuffer1), + 'input2': new MLTensor(inputBuffer2), }; -const outputs = await graph.compute(inputs); +const outputs = graph.compute(inputs); +const data = await outputs.output.data(); // Log the shape and computed result of the output operand. -console.log('Output shape: ' + outputs.output.dimensions); +console.log('Output shape: ' + outputs.output.dimensions()); // Output shape: 1,2,2,2 -console.log('Output value: ' + outputs.output.data); +console.log('Output value: ' + data); // Output value: 2.25,2.25,2.25,2.25,2.25,2.25,2.25,2.25 From f9f02f42a4223df45b40d82c89b97a51842819bf Mon Sep 17 00:00:00 2001 From: Chai Chaoweeraprasit Date: Mon, 12 Apr 2021 22:49:54 -0700 Subject: [PATCH 2/6] Wording update. --- index.bs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/index.bs b/index.bs index c43a4f19..629ed984 100644 --- a/index.bs +++ b/index.bs @@ -454,13 +454,15 @@ They are represented by callbacks and promises in JavaScript. ## Device Selection ## {#programming-model-device-selection} -An {{MLContext}} interface represents a global state of neural network execution. An important context state is the underlying execution device that manages the resources and facilitates the compilation and eventual execution of the neural network graph. {{MLContext}} may be created from a specific GPU device such as {{GPUDevice}} or {{WebGLRenderingContext}} that is already in use by the application, in which case the corresponding {{GPUBuffer}} or {{WebGLBuffer}} resources used as graph constants, as well as the {{GPUTexture}} and {{WebGLTexture}} as graph inputs must also be created from the same device. In a multi-adapter configuration, the device used for {{MLContext}} must be created from the same adapter as the device used to allocate and manage the resources referenced in the graph. If a graph constant or an input is an {{ArrayBufferView}} in the system memory, it is automatically uploaded from the system memory to the GPU memory, and vice versa downloaded to an {{ArrayBufferView}} for a graph output. +An {{MLContext}} interface represents a global state of neural network execution. An important context state is the underlying execution device that manages the resources and facilitates the compilation and the eventual execution of the neural network graph. An {{MLContext}} could be created from a specific GPU device such as {{GPUDevice}} or {{WebGLRenderingContext}} that is already in use by the application, in which case the corresponding {{GPUBuffer}} or {{WebGLBuffer}} resources used as graph constants, as well as the {{GPUTexture}} and {{WebGLTexture}} as graph inputs must also be created from the same device. In a multi-adapter configuration, the device used for {{MLContext}} must be created from the same adapter as the device used to allocate the resources referenced in the graph. -When an {{MLContext}} is created with {{MLContextOptions}}, the user agent selects and creates the underlying execution device by taking into account the application's preference as defined in the {{MLDevicePreference}} option. +In a situation when a GPU context executes a graph with constants or inputs given as {{ArrayBufferView}}, the content of such constant or input is automatically uploaded from the system memory to the GPU memory. Likewise, the content of the execution result is downloaded to the system memory as {{ArrayBufferView}} upon requested. + +When an {{MLContext}} is created with {{MLContextOptions}}, the user agent selects and creates the underlying execution device by taking into account the application's preference specified in the {{MLDevicePreference}} option. - The *"graphics"* device provides the broadest range of achievable performance across hardware platforms from consumer devices to professional workstations. -- The *"compute"* device takes advantage of dedicated compute devices for specific kinds of workload with great power efficiency over a long period of execution time. +- The *"compute"* device takes advantage of special-purpose hardware for specific kinds of workload with great power efficiency over a long period of execution time. - The *"software"* device provides the broadest reach of compute availability, but with limited scalability of execution performance on the more complex neural networks. -- When the device preference is not specified (*"default"*), the user agent selects the most suitable device to use. +- When the device preference is not specified (*"default"*), the user agent automatically selects the most suitable device to use. The following table summarizes all the supported device types and the resource types of constants, inputs, and outputs that each of the device types supports. @@ -511,6 +513,7 @@ enum MLPowerPreference { dictionary MLContextOptions { // Preferred kind of device used MLDevicePreference devicePreference = "default"; + // Preference as related to power consumption MLPowerPreference powerPreference = "default"; }; @@ -1671,7 +1674,7 @@ partial interface MLGraphBuilder { ## MLTensor ## {#api-mltensor} -The {{MLTensor}} interface represents a multidimensional array of numbers with a shape. +The {{MLTensor}} interface represents a resource of a multidimensional array of numbers with a shape. -## MLGraph ## {#api-mlgraph} -The {{MLGraph}} interface represents a compiled computational graph. A compiled graph once constructed is immutable and cannot be subsequently changed. +dictionary MLOutput { + (MLBufferView or WebGLTexture or GPUTexture) data; + sequence dimensions; +}; - @@ -1721,43 +1657,44 @@ interface MLGraph { : \[[implementation]] :: - The underlying implemenation provided by the user agent. + The underlying implemenation provided by the User Agent.
: compute(inputs, outputs) :: - Issue a compute request of the {{MLGraph}} given an input {{MLNamedTensors}} and optional output {{MLNamedTensors}}. The returned {{MLNamedTensors}} are the results ready to be consumed. + Issue a compute request of the {{MLGraph}} given {{MLNamedInputs}} and optional {{MLNamedOutputs}}. The returned {{Promise}} resolves when the results in {{MLNamedOutputs}} are ready to be consumed.
**Called on:** {{MLGraph}} |this|. **Arguments:** 
-                |inputs|: a {{MLNamedTensors}}. The data and optional dimensions of inputs for the compute request.
-                |outputs|: an optional {{MLNamedTensors}}. The names and pre-allocated resources of required outputs for the compute request. Default to be an empty [=record=] which means that the compute request is for all outputs.
+                |inputs|: a {{MLNamedInputs}}. The data and optional dimensions of inputs for the compute request.
+                |outputs|: an optional {{MLNamedOutputs}}. The names and pre-allocated resources of required outputs for the compute request. Default to be an empty [=record=] which means that the compute request is for all outputs.
- **Returns:** {{MLNamedTensors}}. The dimensions and data of outputs returned by the compute request. + **Returns:** {{Promise}}<{{MLNamedOutputs}}>. The dimensions and data of outputs returned by the compute request. + 1. Let |promise| be [=a new promise=]. - 1. If any of the following requirements are unmet, then stop. + 1. If any of the following requirements are unmet, then [=reject=] |promise| with a {{TypeError}} and stop.
1. For each |key| -> |value| of |inputs|: 1. |this|.{{MLGraph/[[inputOperands]]}}[|key|] must exist. 1. Let |inputOperand| be |this|.{{MLGraph/[[inputOperands]]}}[|key|]. - 1. If |value|.{{MLTensor/resource}} is an {{ArrayBufferView}}, then: - 1. The kind of |value|.{{MLTensor/resource}} must be compatible to |inputOperand|.{{MLOperandDescriptor/type}} according to [this table](#appendices-mloperandtype-arraybufferview-compatibility). - 1. If |value|.{{MLTensor/dimensions}} was given, then: - 1. The length of |value|.{{MLTensor/dimensions}} must be the same as the length of |inputOperand|.{{MLOperandDescriptor/dimensions}}. + 1. If |value|.{{MLInput/data}} is an {{ArrayBufferView}}, then: + 1. The kind of |value|.{{MLInput/data}} must be compatible to |inputOperand|.{{MLOperandDescriptor/type}} according to [this table](#appendices-mloperandtype-arraybufferview-compatibility). + 1. If |value|.{{MLInput/dimensions}} was given, then: + 1. The length of |value|.{{MLInput/dimensions}} must be the same as the length of |inputOperand|.{{MLOperandDescriptor/dimensions}}. 1. Let |i| be 0. 1. While true: - 1. Let |dimension| be |value|.{{MLTensor/dimensions}}[|i|]. + 1. Let |dimension| be |value|.{{MLInput/dimensions}}[|i|]. 1. |dimension| must be greater than 0. 1. If |inputOperand|.{{MLOperandDescriptor/dimensions}}[|i|] is greater than 0, then |dimension| must be equal to |inputOperand|.{{MLOperandDescriptor/dimensions}}[|i|]. 1. Set |i| to |i| + 1. - 1. If |i| if equal to the length of |value|.{{MLTensor/dimensions}}, then break. + 1. If |i| if equal to the length of |value|.{{MLInput/dimensions}}, then break. 1. Else: 1. For each |dimension| of |inputOperand|.{{MLOperandDescriptor/dimensions}}: 1. The value of |dimension| must be greater than 0. @@ -1765,7 +1702,7 @@ interface MLGraph { 1. If |outputs| was not an empty [=record=], then: 1. For each |key| -> |value| of |outputs|: 1. |this|.{{MLGraph/[[outputOperands]]}}[|key|] must exist. - 1. If |value|.{{MLTensor/resource}} was given, then the kind of |value|.{{MLTensor/resource}} must be compatible to |this|.{{MLGraph/[[outputOperands]]}}[|key|] according to [this table](#appendices-mloperandtype-arraybufferview-compatibility). + 1. If |value|.{{MLOutput/data}} was given, then the kind of |value|.{{MLOutput/data}} must be compatible to |this|.{{MLGraph/[[outputOperands]]}}[|key|] according to [this table](#appendices-mloperandtype-arraybufferview-compatibility).
1. Let |requiredOutputNames| be a new [=ordered set=]<{{DOMString}}>. @@ -1776,16 +1713,16 @@ interface MLGraph { 1. For each |key| -> |value| of |this|.{{MLGraph/[[outputOperands]]}}: 1. Append |key| to |requiredOutputNames|. - 1. Let |copiedInputs| be a new {{MLNamedTensors}}. + 1. Let |copiedInputs| be a new {{MLNamedInputs}}. 1. For each |key| -> |value| of |inputs|: - 1. Let |copiedInputs| be a new {{MLTensor}}. - 1. Let |copiedInputs|.{{MLTensor/resource}} be a new {{ArrayBufferView}} that has the same kind and length as |value|.{{MLTensor/resource}}'s. - 1. Set the content of |copiedInputs|.{{MLTensor/resource}} to the content of |value|.{{MLTensor/resource}}. - 1. Let |copiedInputs|.{{MLTensor/dimensions}} be a new [=sequence=]<{{long}}> that has the same length of |value|.{{MLTensor/dimensions}}'s. - 1. Set the content of |copiedInputs|.{{MLTensor/dimensions}} to the content of |value|.{{MLTensor/dimensions}}. + 1. Let |copiedInputs| be a new {{MLInput}}. + 1. Let |copiedInputs|.{{MLInput/data}} be a new {{ArrayBufferView}} that has the same kind and length as |value|.{{MLInput/data}}'s. + 1. Set the content of |copiedInputs|.{{MLInput/data}} to the content of |value|.{{MLInput/data}}. + 1. Let |copiedInputs|.{{MLInput/dimensions}} be a new [=sequence=]<{{long}}> that has the same length of |value|.{{MLInput/dimensions}}'s. + 1. Set the content of |copiedInputs|.{{MLInput/dimensions}} to the content of |value|.{{MLInput/dimensions}}. 1. Set |copiedInputs|[key] to |copiedInputs|. - 1. Let |results| be a new {{MLNamedTensors}}. + 1. Let |results| be a new {{MLNamedOutputs}}. 1. Let |remainingOutputNames| be a new [=ordered set=]<{{DOMString}}>. 1. Set the content of |remainingOutputNames| to the content of |requiredOutputNames|. 1. Issue the following steps on the [=Device timeline=] of |this|.{{MLGraph/[[implementation]]}}: @@ -1807,18 +1744,18 @@ interface MLGraph { 1. Set |outputSize| to |outputSize| * |outputDimensions|[|i|]. 1. Set |i| to |i| + 1. 1. If |i| is equal to |outputRank|, then break. - 1. Set |results|[|outputName|].{{MLTensor/dimensions}} to |outputDemisions|. + 1. Set |results|[|outputName|].{{MLOutput/dimensions}} to |outputDemisions|. 1. If |this|.{{MLGraph/[[context]]}} is created from {{MLContextOptions}}, then: - 1. If |outputs|[|outputName|].{{MLTensor/resource}} was given, then: - 1. If outputs|[|outputName|].{{MLTensor/resource}} is not an {{ArrayBufferView}}, then stop. - 1. If the kind of |outputs|[|outputName|].{{MLTensor/resource}} is not compatible to output tensor according to [this table](#appendices-mloperandtype-arraybufferview-compatibility), then stop. - 1. If the length of |outputs|[|outputName|].{{MLTensor/resource}} is less than |outputSize|, then stop. - 1. Set the content of |outputs|[|outputName|].{{MLTensor/resource}} to the content of output tensor returned by |this|.{{MLGraph/[[implementation]]}}. + 1. If |outputs|[|outputName|].{{MLOutput/data}} was given, then: + 1. If outputs|[|outputName|].{{MLOutput/data}} is not an {{ArrayBufferView}}, then [=reject=] |promise| with an {{TypeError}} and stop. + 1. If the kind of |outputs|[|outputName|].{{MLOutput/data}} is not compatible to output tensor according to [this table](#appendices-mloperandtype-arraybufferview-compatibility), then [=reject=] |promise| with a {{TypeError}} and stop. + 1. If the length of |outputs|[|outputName|].{{MLOutput/data}} is less than |outputSize|, then [=reject=] |promise| with a {{TypeError}} and stop. + 1. Set the content of |outputs|[|outputName|].{{MLOutput/data}} to the content of output tensor returned by |this|.{{MLGraph/[[implementation]]}}. 1. Else: - 1. Let |results|[|outputName|].{{MLTensor/resource}} be a new {{ArrayBufferView}} of size |outputSize| and kind that is compatible to output tensor according to [this table](#appendices-mloperandtype-arraybufferview-compatibility). - 1. Set the content of |results|[|outputName|].{{MLTensor/resource}} to the content of output tensor returned by |this|.{{MLGraph/[[implementation]]}}. + 1. Let |results|[|outputName|].{{MLOutput/data}} be a new {{ArrayBufferView}} of size |outputSize| and kind that is compatible to output tensor according to [this table](#appendices-mloperandtype-arraybufferview-compatibility). + 1. Set the content of |results|[|outputName|].{{MLOutput/data}} to the content of output tensor returned by |this|.{{MLGraph/[[implementation]]}}. 1. Remove |outputName| from |remainingOutputNames|. - 1. If |remainingOutputNames| is empty, then stop. + 1. If |remainingOutputNames| is empty, then resolve |promise| with |results| and stop.
@@ -1850,12 +1787,11 @@ async function compute(shapeA, shapeB) { // Specify the shape of inputs when computing. const inputs = { - 'a': new MLTensor(bufferA, shapeA), - 'b': new MLTensor(bufferB, shapeB) + 'a': {data: bufferA, dimensions: shapeA}, + 'b': {data: bufferB, dimensions: shapeB}, }; - const outputs = graph.compute(inputs); - const data = await outputs.c.data(); - console.log(`shape: [${outputs.c.dimensions()}], values: ${data}`); + const outputs = await graph.compute(inputs); + console.log(`shape: [${outputs.c.dimensions}], values: ${outputs.c.data}`); } await compute([3, 4], [4, 3]); @@ -1881,12 +1817,11 @@ const c = builder.matmul(a, b); const graph = await builder.build({c}); const bufferA = new Float32Array(sizeOfShape(descA.dimensions)).fill(0.5); -const inputs = {'a': new MLTensor(bufferA)}; +const inputs = {'a': {data: bufferA}}; // Pre-allocate output buffer for c. -const outputs = {'c': new MLTensor(new Float32Array(sizeOfShape([3, 3])))}; -graph.compute(inputs, outputs); -const data = await outputs.c.data(); -console.log(`values: ${data}`); +const outputs = {'c': {data: new Float32Array(sizeOfShape([3, 3]))}}; +await graph.compute(inputs, outputs); +console.log(`values: ${outputs.c.data}`); @@ -1910,23 +1845,21 @@ const e = builder.add(d, c); const graph = await builder.build({d, e}); const bufferA = new Float32Array(sizeOfShape(descA.dimensions)).fill(0.5); -const inputs = {'a': new MLTensor(bufferA)}; +const inputs = {'a': {data: bufferA}}; // Compute both d and e. -let outputs = graph.compute(inputs); +let outputs = await graph.compute(inputs); console.log(`outputs include ${Object.keys(outputs)}`); // Compute d. -outputs = graph.compute(inputs, {d}); -let data = await outputs.d.data(); +outputs = await graph.compute(inputs, {d}); console.log(`outputs include ${Object.keys(outputs)}`); -console.log(`shape: [${outputs.d.dimensions()}], values: ${data}`); +console.log(`shape: [${outputs.d.dimensions}], values: ${outputs.d.data}`); // Compute e. -outputs = graph.compute(inputs, {e}); -data = await outputs.e.data(); +outputs = await graph.compute(inputs, {e}); console.log(`outputs include ${Object.keys(outputs)}`); -console.log(`shape: [${outputs.e.dimensions()}], values: ${data}`); +console.log(`shape: [${outputs.e.dimensions}], values: ${outputs.e.data}`); @@ -2001,18 +1934,17 @@ The following code executes the compiled graph. const inputBuffer1 = new Float32Array(TENSOR_SIZE).fill(1); const inputBuffer2 = new Float32Array(TENSOR_SIZE).fill(1); -// Execute the compiled graph with the specified inputs and asynchronously download the result. +// Asynchronously execute the compiled graph with the specified inputs. const inputs = { - 'input1': new MLTensor(inputBuffer1), - 'input2': new MLTensor(inputBuffer2), + 'input1': {data: inputBuffer1}, + 'input2': {data: inputBuffer2}, }; -const outputs = graph.compute(inputs); -const data = await outputs.output.data(); +const outputs = await graph.compute(inputs); // Log the shape and computed result of the output operand. -console.log('Output shape: ' + outputs.output.dimensions()); +console.log('Output shape: ' + outputs.output.dimensions); // Output shape: 1,2,2,2 -console.log('Output value: ' + data); +console.log('Output value: ' + outputs.output.data); // Output value: 2.25,2.25,2.25,2.25,2.25,2.25,2.25,2.25 From 59edff8730eed1740e08ef0127b55be182f9e914 Mon Sep 17 00:00:00 2001 From: Chai Chaoweeraprasit Date: Sat, 1 May 2021 14:26:39 -0700 Subject: [PATCH 5/6] Device preference and device selection logic. --- index.bs | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/index.bs b/index.bs index 53acd0b6..a824d744 100644 --- a/index.bs +++ b/index.bs @@ -201,6 +201,20 @@ thead.stickyheader th, th.stickyheader { background: var(--stickyheader-background); } +/* + * Generic table format. + */ +th { + text-align: left; +} + +th, td { + border-bottom: 1px solid black; + border-collapse: collapse; + padding-left: 5px; + padding-right: 5px; +} + /* * Darkmode colors */ @@ -435,6 +449,30 @@ They are represented by callbacks and promises in JavaScript. +## Device Selection ## {#programming-model-device-selection} + +An {{MLContext}} interface represents a global state of neural network execution. One of the important context states is the underlying execution device that manages the resources and facilitates the compilation and the eventual execution of the neural network graph. An {{MLContext}} could be created from a specific GPU device such as {{GPUDevice}} or {{WebGLRenderingContext}} that is already in use by the application, in which case the corresponding {{GPUBuffer}} or {{WebGLBuffer}} resources used as graph constants, as well as the {{GPUTexture}} and {{WebGLTexture}} as graph inputs must also be created from the same device. In a multi-adapter configuration, the device used for {{MLContext}} must be created from the same adapter as the device used to allocate the resources referenced in the graph. + +In a situation when a GPU context executes a graph with constants or inputs given as {{ArrayBufferView}}, the content of such constant or input is automatically uploaded from the system memory to the GPU memory. Likewise, the content of the execution result is downloaded to the system memory as {{ArrayBufferView}} upon requested. + +When an {{MLContext}} is created with {{MLContextOptions}}, the user agent selects and creates the underlying execution device by taking into account the application's preference specified in the {{MLPowerPreference}} and the {{MLDevicePreference}} options: +- The *"gpu"* device provides the broadest range of achievable performance across graphics hardware platforms from consumer devices to professional workstations. +- The *"cpu"* device provides the broadest reach of software compute availability, but with limited scalability of execution performance on the more complex neural networks. +- When the device preference is not specified (*"default"*), the user agent selects the most suitable device to use. + +The following table summarizes the types of resource supported by the device selected. + +
+ +
Device TypeArrayBufferViewGPUBufferGPUTextureWebGLBufferWebGLTexture +
GPUDeviceYesYesYesNoNo +
WebGLRenderingContextYesNoNoYesYes +
defaultYesNoNoNoNo +
gpuYesNoNoNoNo +
cpuYesNoNoNoNo +
+
+ API {#api} ===================== @@ -447,16 +485,27 @@ partial interface Navigator { ## ML ## {#api-ml} + + ## MLContext ## {#api-mlcontext} The {{MLContext}} interface represents a global state of neural network compute workload and execution processes.