From 12f10ea93e93e6587b38e0c14a7239f49931744c Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 5 Apr 2023 15:18:40 +0200 Subject: [PATCH 1/9] Add first draft of the docs to extending_types.rst --- docs/source/python/extending_types.rst | 149 +++++++++++++++++++++++++ 1 file changed, 149 insertions(+) diff --git a/docs/source/python/extending_types.rst b/docs/source/python/extending_types.rst index 9b6743cb102..c5db7d5e93e 100644 --- a/docs/source/python/extending_types.rst +++ b/docs/source/python/extending_types.rst @@ -357,3 +357,152 @@ pandas ``ExtensionArray``. This method should have the following signature:: This way, you can control the conversion of a pyarrow ``Array`` of your pyarrow extension type to a pandas ``ExtensionArray`` that can be stored in a DataFrame. + + +Canonical extension types +~~~~~~~~~~~~~~~~~~~~~~~~~ + +In the :ref:`format_canonical_extensions` under the section **Official List** +there is a list of canonical extension types. Here you can find an example of how +to implement and how to use the listed canonical extension types. + +Fixed size tensor +""""""""""""""""" + +Define fixed shape tensor extension type: + +.. code-block:: python + + >>> tensor_type = pa.fixed_shape_tensor(pa.int32(), (2, 2)) + +Create an array of tensors with storage array and defined fixed shape tensor +extension type: + +.. code-block:: python + + >>> arr = [[1, 2, 3, 4], [10, 20, 30, 40], [100, 200, 300, 400]] + >>> storage = pa.array(arr, pa.list_(pa.int32(), 4)) + >>> tensor = pa.ExtensionArray.from_storage(tensor_type, storage) + +Create another array of tensors with different value type: + +.. code-block:: python + + >>> tensor_type2 = pa.fixed_shape_tensor(pa.float32(), (2, 2)) + >>> storage2 = pa.array(arr, pa.list_(pa.float32(), 4)) + >>> tensor2 = pa.ExtensionArray.from_storage(tensor_type2, storage2) + +Create a ``pyarrow.Table`` with random data and two tensor arrays: + +.. code-block:: python + + >>> data = [ + ... pa.array([1, 2, 3]), + ... pa.array(['foo', 'bar', None]), + ... pa.array([True, None, True]), + ... tensor, + ... tensor2 + ... ] + >>> my_schema = pa.schema([('f0', pa.int8()), + ... ('f1', pa.string()), + ... ('f2', pa.bool_()), + ... ('tensors_int', tensor_type), + ... ('tensors_float', tensor_type2)]) + >>> table = pa.Table.from_arrays(data, schema=my_schema) + >>> table + pyarrow.Table + f0: int8 + f1: string + f2: bool + tensors_int: extension + tensors_float: extension + ---- + f0: [[1,2,3]] + f1: [["foo","bar",null]] + f2: [[true,null,true]] + tensors_int: [[[1,2,3,4],[10,20,30,40],[100,200,300,400]]] + tensors_float: [[[1,2,3,4],[10,20,30,40],[100,200,300,400]]] + +Convert a tensor array to numpy ndarray (tensor): + +.. code-block:: python + + >>> numpy_tensor = tensor2.to_numpy_ndarray() + >>> numpy_tensor + array([[[ 1., 2.], + [ 3., 4.]], + [[ 10., 20.], + [ 30., 40.]], + [[100., 200.], + [300., 400.]]]) + +Convert a list of numpy ndarrays (tensors) to a tensor array: + +.. code-block:: python + + >>> pa.FixedShapeTensorArray.from_numpy_ndarray(numpy_tensor) + + [ + [ + 1, + 2, + 3, + 4 + ], + [ + 10, + 20, + 30, + 40 + ], + [ + 100, + 200, + 300, + 400 + ] + ] + +Example of using permutation parameter when converting to numpy ndarray: + +.. code-block:: python + + >>> tensor_type = pa.fixed_shape_tensor(pa.int32(), (2, 2), permutation=[1,0]) + >>> arr = [[1, 2, 3, 4], [10, 20, 30, 40], [100, 200, 300, 400]] + >>> storage = pa.array(arr, pa.list_(pa.int32(), 4)) + >>> tensor = pa.ExtensionArray.from_storage(tensor_type, storage) + +Converting tensor to ndarray without the use of permutation parameter: + +.. code-block:: python + + >>> tensor.to_numpy_ndarray() + array([[[ 1, 2], + [ 3, 4]], + [[ 10, 20], + [ 30, 40]], + [[100, 200], + [300, 400]]], dtype=int32) + +Converting tensor to ndarray with permutation parameter needs recalculation of +the permutation parameter. + +The dimension of the ndarray is one dimension higher as the first dimension is +always the array of tensors. For that we need to increment the values of the +permutation parameter and add the first dimension (dimension of the array of tensors): + +.. code-block:: python + + >>> permutation = [x+1 for x in tensor.type.permutation] + >>> permutation + [2, 1] + >>> permutation.insert(0,0) + >>> permutation + [0, 2, 1] + >>> tensor.to_numpy_ndarray().transpose(permutation) + array([[[ 1, 3], + [ 2, 4]], + [[ 10, 30], + [ 20, 40]], + [[100, 300], + [200, 400]]], dtype=int32) From ae6bdf41c23f2ba9fc168c3348087418810fec0f Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 7 Apr 2023 09:20:00 +0200 Subject: [PATCH 2/9] Refine the description of permutation parameter and add example of NCHW to NHWC format --- docs/source/python/extending_types.rst | 38 +++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/docs/source/python/extending_types.rst b/docs/source/python/extending_types.rst index c5db7d5e93e..b97835b2ed9 100644 --- a/docs/source/python/extending_types.rst +++ b/docs/source/python/extending_types.rst @@ -463,7 +463,13 @@ Convert a list of numpy ndarrays (tensors) to a tensor array: ] ] -Example of using permutation parameter when converting to numpy ndarray: +Both optional parameters, ``permutation`` and ``dim_names`` are meant to provide the user +with the information about the logical layout of the data compared to the physical layout +and are **not used** when converting fixed shape tensor extension array to numpy ndarray. + +If the user needs to use the information of the optional parameters on numpy ndarrays, +for example to revert the logical view to be equal to the physical view, one could do +the following: .. code-block:: python @@ -472,7 +478,8 @@ Example of using permutation parameter when converting to numpy ndarray: >>> storage = pa.array(arr, pa.list_(pa.int32(), 4)) >>> tensor = pa.ExtensionArray.from_storage(tensor_type, storage) -Converting tensor to ndarray without the use of permutation parameter: +First you convert the tensor array to ndarray (``permutation`` is not used +in the conversion): .. code-block:: python @@ -484,13 +491,19 @@ Converting tensor to ndarray without the use of permutation parameter: [[100, 200], [300, 400]]], dtype=int32) -Converting tensor to ndarray with permutation parameter needs recalculation of -the permutation parameter. +Before using the permutation parameter to transpose the numpy ndarray we received, +we first need to do some small recalculation. The dimension of the ndarray is one dimension higher as the first dimension is always the array of tensors. For that we need to increment the values of the permutation parameter and add the first dimension (dimension of the array of tensors): +.. note:: + + The first dimension we are adding should always equal ``0`` due to the design of + the storage type of the extension, which is ``FixedSizeList`` and where the first + dimension is the array of individual tensors with given shape. + .. code-block:: python >>> permutation = [x+1 for x in tensor.type.permutation] @@ -506,3 +519,20 @@ permutation parameter and add the first dimension (dimension of the array of ten [ 20, 40]], [[100, 300], [200, 400]]], dtype=int32) + +In case we have a tensor array with ``dim_names`` parameter defined we can use +permutation to change the logical layout. For example, if we have a tensor array +with ``dim_names=['C', 'H', 'W']`` which is ``NCHW`` format where: + +* N: number of images which is in our case the length of an array +* H: height of the image +* W: width of the image +* C: number of channels of the image + +and we want to transpose the numpy ndarray to use ``NHWC``format, we could use +permutations as we have above. The permutation array would in this case be +``[0, 2, 3, 1]`` where the first dimension will stay same, the second dimension +will be the third dimension of the original layout (which is the second element in +``dim_names``), the third dimension will be the forth dimension of the original +layout (which is the third element in ``dim_names``) and the last dimension will +be the second dimension of the original layout (first element of ``dim_names``). From e591587cb5e5f7d3261756a1b60917ad533cd14a Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 11 Apr 2023 14:35:22 +0200 Subject: [PATCH 3/9] Update examples to no use non-trivial permutation --- docs/source/python/extending_types.rst | 74 ++------------------------ 1 file changed, 4 insertions(+), 70 deletions(-) diff --git a/docs/source/python/extending_types.rst b/docs/source/python/extending_types.rst index b97835b2ed9..3f7d95089eb 100644 --- a/docs/source/python/extending_types.rst +++ b/docs/source/python/extending_types.rst @@ -463,76 +463,10 @@ Convert a list of numpy ndarrays (tensors) to a tensor array: ] ] -Both optional parameters, ``permutation`` and ``dim_names`` are meant to provide the user -with the information about the logical layout of the data compared to the physical layout -and are **not used** when converting fixed shape tensor extension array to numpy ndarray. - -If the user needs to use the information of the optional parameters on numpy ndarrays, -for example to revert the logical view to be equal to the physical view, one could do -the following: - -.. code-block:: python - - >>> tensor_type = pa.fixed_shape_tensor(pa.int32(), (2, 2), permutation=[1,0]) - >>> arr = [[1, 2, 3, 4], [10, 20, 30, 40], [100, 200, 300, 400]] - >>> storage = pa.array(arr, pa.list_(pa.int32(), 4)) - >>> tensor = pa.ExtensionArray.from_storage(tensor_type, storage) - -First you convert the tensor array to ndarray (``permutation`` is not used -in the conversion): - -.. code-block:: python - - >>> tensor.to_numpy_ndarray() - array([[[ 1, 2], - [ 3, 4]], - [[ 10, 20], - [ 30, 40]], - [[100, 200], - [300, 400]]], dtype=int32) - -Before using the permutation parameter to transpose the numpy ndarray we received, -we first need to do some small recalculation. - -The dimension of the ndarray is one dimension higher as the first dimension is -always the array of tensors. For that we need to increment the values of the -permutation parameter and add the first dimension (dimension of the array of tensors): - .. note:: - The first dimension we are adding should always equal ``0`` due to the design of - the storage type of the extension, which is ``FixedSizeList`` and where the first - dimension is the array of individual tensors with given shape. - -.. code-block:: python + Both optional parameters, ``permutation`` and ``dim_names``, are meant to provide the user + with the information about the logical layout of the data compared to the physical layout. - >>> permutation = [x+1 for x in tensor.type.permutation] - >>> permutation - [2, 1] - >>> permutation.insert(0,0) - >>> permutation - [0, 2, 1] - >>> tensor.to_numpy_ndarray().transpose(permutation) - array([[[ 1, 3], - [ 2, 4]], - [[ 10, 30], - [ 20, 40]], - [[100, 300], - [200, 400]]], dtype=int32) - -In case we have a tensor array with ``dim_names`` parameter defined we can use -permutation to change the logical layout. For example, if we have a tensor array -with ``dim_names=['C', 'H', 'W']`` which is ``NCHW`` format where: - -* N: number of images which is in our case the length of an array -* H: height of the image -* W: width of the image -* C: number of channels of the image - -and we want to transpose the numpy ndarray to use ``NHWC``format, we could use -permutations as we have above. The permutation array would in this case be -``[0, 2, 3, 1]`` where the first dimension will stay same, the second dimension -will be the third dimension of the original layout (which is the second element in -``dim_names``), the third dimension will be the forth dimension of the original -layout (which is the third element in ``dim_names``) and the last dimension will -be the second dimension of the original layout (first element of ``dim_names``). + The conversion to numpy ndarray is only possible for trivial permutations (``None`` or + ``[0, 1, ... N-1]`` where ``N`` is the number of tensor dimensions). From 80bdbf49819c21c7e75433f3b11b0941111035a8 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 11 Apr 2023 18:24:31 +0200 Subject: [PATCH 4/9] Apply suggestions from code review - rok Co-authored-by: Rok Mihevc --- docs/source/python/extending_types.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/python/extending_types.rst b/docs/source/python/extending_types.rst index 3f7d95089eb..f2775b3f15d 100644 --- a/docs/source/python/extending_types.rst +++ b/docs/source/python/extending_types.rst @@ -363,8 +363,8 @@ Canonical extension types ~~~~~~~~~~~~~~~~~~~~~~~~~ In the :ref:`format_canonical_extensions` under the section **Official List** -there is a list of canonical extension types. Here you can find an example of how -to implement and how to use the listed canonical extension types. +there is a list of canonical extension types. Here we list examples of how to +use the listed canonical extension types. Fixed size tensor """"""""""""""""" @@ -382,7 +382,7 @@ extension type: >>> arr = [[1, 2, 3, 4], [10, 20, 30, 40], [100, 200, 300, 400]] >>> storage = pa.array(arr, pa.list_(pa.int32(), 4)) - >>> tensor = pa.ExtensionArray.from_storage(tensor_type, storage) + >>> tensor_array = pa.ExtensionArray.from_storage(tensor_type, storage) Create another array of tensors with different value type: From 9c63049f4e490f765aedc049a731a3e34c004027 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 11 Apr 2023 18:33:57 +0200 Subject: [PATCH 5/9] Change tensor to tensor_array --- docs/source/python/extending_types.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/python/extending_types.rst b/docs/source/python/extending_types.rst index f2775b3f15d..50b6bb836b6 100644 --- a/docs/source/python/extending_types.rst +++ b/docs/source/python/extending_types.rst @@ -388,9 +388,9 @@ Create another array of tensors with different value type: .. code-block:: python - >>> tensor_type2 = pa.fixed_shape_tensor(pa.float32(), (2, 2)) + >>> tensor_type_2 = pa.fixed_shape_tensor(pa.float32(), (2, 2)) >>> storage2 = pa.array(arr, pa.list_(pa.float32(), 4)) - >>> tensor2 = pa.ExtensionArray.from_storage(tensor_type2, storage2) + >>> tensor_array_2 = pa.ExtensionArray.from_storage(tensor_type_2, storage2) Create a ``pyarrow.Table`` with random data and two tensor arrays: @@ -400,14 +400,14 @@ Create a ``pyarrow.Table`` with random data and two tensor arrays: ... pa.array([1, 2, 3]), ... pa.array(['foo', 'bar', None]), ... pa.array([True, None, True]), - ... tensor, - ... tensor2 + ... tensor_array, + ... tensor_array_2 ... ] >>> my_schema = pa.schema([('f0', pa.int8()), ... ('f1', pa.string()), ... ('f2', pa.bool_()), ... ('tensors_int', tensor_type), - ... ('tensors_float', tensor_type2)]) + ... ('tensors_float', tensor_type_2)]) >>> table = pa.Table.from_arrays(data, schema=my_schema) >>> table pyarrow.Table @@ -427,7 +427,7 @@ Convert a tensor array to numpy ndarray (tensor): .. code-block:: python - >>> numpy_tensor = tensor2.to_numpy_ndarray() + >>> numpy_tensor = tensor_array_2.to_numpy_ndarray() >>> numpy_tensor array([[[ 1., 2.], [ 3., 4.]], From 575ad5e42a0f3e565ecc2cd4cbc52123f4f4c810 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 12 Apr 2023 06:09:21 +0200 Subject: [PATCH 6/9] Add more text and rearange --- docs/source/python/extending_types.rst | 61 +++++++++++++++++++------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/docs/source/python/extending_types.rst b/docs/source/python/extending_types.rst index 50b6bb836b6..d02a3679906 100644 --- a/docs/source/python/extending_types.rst +++ b/docs/source/python/extending_types.rst @@ -362,21 +362,25 @@ extension type to a pandas ``ExtensionArray`` that can be stored in a DataFrame. Canonical extension types ~~~~~~~~~~~~~~~~~~~~~~~~~ -In the :ref:`format_canonical_extensions` under the section **Official List** -there is a list of canonical extension types. Here we list examples of how to -use the listed canonical extension types. +You can find the official list of canonical extension types in the +:ref:`format_canonical_extensions` section. Here we add examples on how to +use them in pyarrow. Fixed size tensor """"""""""""""""" -Define fixed shape tensor extension type: +To create an array of tensors with equal shape (fixed shape tensor array) we +first need to define a fixed shape tensor extension type with value type +and shape: .. code-block:: python >>> tensor_type = pa.fixed_shape_tensor(pa.int32(), (2, 2)) -Create an array of tensors with storage array and defined fixed shape tensor -extension type: +Then we need the storage array with :func:`pyarrow.list_` type where ``value_type``` +is the fixed shape tensor value type and list size is a product of ``tensor_type`` +shape elements. Then we can create an array of tensors with +``pa.ExtensionArray.from_storage()`` method: .. code-block:: python @@ -384,15 +388,16 @@ extension type: >>> storage = pa.array(arr, pa.list_(pa.int32(), 4)) >>> tensor_array = pa.ExtensionArray.from_storage(tensor_type, storage) -Create another array of tensors with different value type: +We can also create another array of tensors with different value type: .. code-block:: python >>> tensor_type_2 = pa.fixed_shape_tensor(pa.float32(), (2, 2)) - >>> storage2 = pa.array(arr, pa.list_(pa.float32(), 4)) - >>> tensor_array_2 = pa.ExtensionArray.from_storage(tensor_type_2, storage2) + >>> storage_2 = pa.array(arr, pa.list_(pa.float32(), 4)) + >>> tensor_array_2 = pa.ExtensionArray.from_storage(tensor_type_2, storage_2) -Create a ``pyarrow.Table`` with random data and two tensor arrays: +Extension arrays can be used as columns in ``pyarrow.Table`` or +``pyarrow.RecordBatch``: .. code-block:: python @@ -423,7 +428,7 @@ Create a ``pyarrow.Table`` with random data and two tensor arrays: tensors_int: [[[1,2,3,4],[10,20,30,40],[100,200,300,400]]] tensors_float: [[[1,2,3,4],[10,20,30,40],[100,200,300,400]]] -Convert a tensor array to numpy ndarray (tensor): +We can also convert a tensor array to a numpy ndarray: .. code-block:: python @@ -436,7 +441,16 @@ Convert a tensor array to numpy ndarray (tensor): [[100., 200.], [300., 400.]]]) -Convert a list of numpy ndarrays (tensors) to a tensor array: +.. note:: + + Both optional parameters, ``permutation`` and ``dim_names``, are meant to provide the user + with the information about the logical layout of the data compared to the physical layout. + + The conversion to numpy ndarray is only possible for trivial permutations (``None`` or + ``[0, 1, ... N-1]`` where ``N`` is the number of tensor dimensions). + +And also the other way around, we can convert a list of numpy ndarrays to a fixed shape tensor +array: .. code-block:: python @@ -463,10 +477,23 @@ Convert a list of numpy ndarrays (tensors) to a tensor array: ] ] -.. note:: +The extension type can also have ``permutation`` and ``dim_names`` defined. For +example - Both optional parameters, ``permutation`` and ``dim_names``, are meant to provide the user - with the information about the logical layout of the data compared to the physical layout. +.. code-block:: python - The conversion to numpy ndarray is only possible for trivial permutations (``None`` or - ``[0, 1, ... N-1]`` where ``N`` is the number of tensor dimensions). + >>> tensor_type = pa.fixed_shape_tensor(pa.float64(), [2, 2, 3], permutation=[0, 2, 1]) + +or + +.. code-block:: python + + >>> tensor_type = pa.fixed_shape_tensor(pa.bool_(), [2, 2, 3], dim_names=['C', 'H', 'W']) + +for ``NCHW`` format where: + +* N: number of images which is in our case the length of an array and is always on + the first dimension +* C: number of channels of the image +* H: height of the image +* W: width of the image From 9c1e0f4ad34fc1b4060b5f7abe39f7b3bf584f06 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 12 Apr 2023 09:34:18 +0200 Subject: [PATCH 7/9] Correct alignment and add info about shape change in conversion numpy->pyarrow --- docs/source/python/extending_types.rst | 27 +++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/source/python/extending_types.rst b/docs/source/python/extending_types.rst index d02a3679906..13eebef41d4 100644 --- a/docs/source/python/extending_types.rst +++ b/docs/source/python/extending_types.rst @@ -435,11 +435,11 @@ We can also convert a tensor array to a numpy ndarray: >>> numpy_tensor = tensor_array_2.to_numpy_ndarray() >>> numpy_tensor array([[[ 1., 2.], - [ 3., 4.]], - [[ 10., 20.], - [ 30., 40.]], - [[100., 200.], - [300., 400.]]]) + [ 3., 4.]], + [[ 10., 20.], + [ 30., 40.]], + [[100., 200.], + [300., 400.]]]) .. note:: @@ -477,6 +477,23 @@ array: ] ] +With the conversion the first dimension of the ndarray becomes the length of the pyarrow extension +array. We can see in the example that ndarray of shape ``(3, 2, 2)`` becomes an arrow array of +length 3 with tensor elements of shape ``(2, 2)``. + +.. code-block:: python + + # ndarray of shape (3, 2, 2) + >>> numpy_tensor.shape + (3, 2, 2) + + # arrow array of length 3 with tensor elements of shape (2, 2) + >>> pyarrow_tensor_array = pa.FixedShapeTensorArray.from_numpy_ndarray(numpy_tensor) + >>> len(pyarrow_tensor_array) + 3 + >>> pyarrow_tensor_array.type.shape + [2, 2] + The extension type can also have ``permutation`` and ``dim_names`` defined. For example From b3e3a57fdebed0c5ca7c22dbfa98aacc5d198a04 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 12 Apr 2023 09:35:22 +0200 Subject: [PATCH 8/9] Reapply code review changes (text) --- docs/source/python/extending_types.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/source/python/extending_types.rst b/docs/source/python/extending_types.rst index 13eebef41d4..da28b1a4aa6 100644 --- a/docs/source/python/extending_types.rst +++ b/docs/source/python/extending_types.rst @@ -428,7 +428,7 @@ Extension arrays can be used as columns in ``pyarrow.Table`` or tensors_int: [[[1,2,3,4],[10,20,30,40],[100,200,300,400]]] tensors_float: [[[1,2,3,4],[10,20,30,40],[100,200,300,400]]] -We can also convert a tensor array to a numpy ndarray: +We can also convert a tensor array to a single multi-dimensional numpy ndarray: .. code-block:: python @@ -449,8 +449,7 @@ We can also convert a tensor array to a numpy ndarray: The conversion to numpy ndarray is only possible for trivial permutations (``None`` or ``[0, 1, ... N-1]`` where ``N`` is the number of tensor dimensions). -And also the other way around, we can convert a list of numpy ndarrays to a fixed shape tensor -array: +And also the other way around, we can convert a numpy ndarray to a fixed shape tensor array: .. code-block:: python From cd10fbfeaf902463b56c9a9761737d11b77cb48f Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 12 Apr 2023 09:42:45 +0200 Subject: [PATCH 9/9] Add text for the shape change in pyarrow->numpy conversion also --- docs/source/python/extending_types.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/source/python/extending_types.rst b/docs/source/python/extending_types.rst index da28b1a4aa6..53ce70e13b4 100644 --- a/docs/source/python/extending_types.rst +++ b/docs/source/python/extending_types.rst @@ -428,7 +428,9 @@ Extension arrays can be used as columns in ``pyarrow.Table`` or tensors_int: [[[1,2,3,4],[10,20,30,40],[100,200,300,400]]] tensors_float: [[[1,2,3,4],[10,20,30,40],[100,200,300,400]]] -We can also convert a tensor array to a single multi-dimensional numpy ndarray: +We can also convert a tensor array to a single multi-dimensional numpy ndarray. +With the conversion the length of the arrow array becomes the first dimension +in the numpy ndarray: .. code-block:: python @@ -440,6 +442,8 @@ We can also convert a tensor array to a single multi-dimensional numpy ndarray: [ 30., 40.]], [[100., 200.], [300., 400.]]]) + >>> numpy_tensor.shape + (3, 2, 2) .. note::