From df1ffa932dc582a0a99e9b67f4682ac835c73776 Mon Sep 17 00:00:00 2001 From: Manupa Karunaratne Date: Tue, 17 Aug 2021 17:27:51 +0100 Subject: [PATCH 1/7] [RFC][TIR] TIR Pinned Memory Representation * adding the markdown Change-Id: I6980d8f9a228ff5e8a79d74220db9b77f88a3e1b --- ...ing-allocate-nodes-with-pinned-memories.md | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 rfcs/000x-associating-allocate-nodes-with-pinned-memories.md diff --git a/rfcs/000x-associating-allocate-nodes-with-pinned-memories.md b/rfcs/000x-associating-allocate-nodes-with-pinned-memories.md new file mode 100644 index 00000000..b56a54e0 --- /dev/null +++ b/rfcs/000x-associating-allocate-nodes-with-pinned-memories.md @@ -0,0 +1,158 @@ + +- Feature Name: TIR Pinned Memory Representation +- Start Date: 2021-06-01 +- RFC PR: TBD +- GitHub Issue: TBD + +# 1. Summary + +This RFC proposes how pinned memories could be associated with tir.allocate nodes (and allocate_const nodes) and used by passes in the lowering process. + +# 2. Motivation + +Currently, TVM relies on dynamic (alloc and free style) allocations in runtime to manage the intermediary memory used by operators and the network. This is sometimes not desirable, especially in microTVM. + +The current design of [Unified Static Memory Planner (USMP)](https://github.com/apache/tvm-rfcs/pull/9), enables the user option to provide buffers to place workspace and constant tensors. + +``` + tvmc compile my_model.tflite + --executor=aot + --target=accel,c + --with-workspace-buffer= "name=dtcm;target=c;size=1000" # Here the size is more of a hint/guide provided to USMP + --with-workspace-buffer= "name=sram;target=c,accel" + --with-parameter-buffer= "name=itcm;target=c;size=5000" # Here the size is more of a hint/guide provided to USMP + --with-parameter-buffer= "name=flash;target=c,accel" +``` + +``` + // The User Application + extern const TVMModel my_model; + __attribute__((section( "ITCM" ) const uint8_t my_model_params_1[TVM_MY_MODEL_ITCM_PARAMETER_BUFFER_SIZE] = ; + __attribute__((section( "FLASH" ), aligned( 16 ))) const uint8_t my_model_params_2[TVM_MY_MODEL_FLASH_PARAMETER_BUFFER_SIZE] = ; + __attribute__((section( "DTCM" ) static uint8_t workspace_buffer_1[TVM_MY_MODEL_DTCM_WORKSPACE_BUFFER_SIZE]; + __attribute__((section( "SRAM" ), aligned( 16 ))) static uint8_t workspace_buffer_2[TVM_MY_MODEL_SRAM_WORKSPACE_BUFFER_SIZE]; + + int main(...) { + ... + TVMContext context; + TVMInputs_my_model_1 inputs = {input}; + TVMOutputs_my_model_1 outputs = {output}; + TVMWorkspaces_my_model workspaces = { + .sram = &workspace_buffer_1, + .dtcm = &workspace_buffer_2, + }; + TVMParameters_my_model parameters = { + .flash = &my_model_params_1, + .itcm = &my_model_params_2 + }; + TVMSetWorkspaces(&context, &workspaces); + TVMSetParameters(&context, parameters); + TVMExecute(&my_model, &inputs, &outputs, &context); + } +``` + +Therefore, we'd need a way to represent the association of each of these memories, that the user will pin the buffers to, closer to allocate nodes in TIR. + +# 3. Guide-level explanation + +This is not particularly a user-facing feature. + +However, the intention here is to associate the name of the memory (along with other information) closer to the allocate IR node. Therefore at the end of the compilation, metadata module (+ header) will generate pointer structs to-be passed in the from the application layer. + + In the case where user does not wish to pass in workspace or constant buffers, metadata module will generate a workspace buffer and a constant buffer, as explained in U1 of [USMP](https://github.com/apache/tvm-rfcs/pull/9). + + # 4. Reference-level explanation + + At the IR, we ll need to associate each allocate node with one (or more) memories that it can end up, because the scheduling might be satisfied with placing buffers in any of the memories in a given set of memories. Therefore, the scheduler might want the memory planners to decide which memory to use based on finding the allocation that fit. + + ## S1. Using AttrStmt to associate additional info : + +TIR: + ``` +allocate_node_1 = tir.allocate([157323], "int16", "global") +tir.attr(allocate_node_1, "pinned_memory", "foo_memory,bar_memory") + ``` + +## S2. Directly as an allocate node argument : + + ``` +class AllocateNode : public StmtNode { + public: + /*! \brief The buffer variable. */ + Var buffer_var; + /*! \brief The type of the buffer. */ + DataType dtype; + /*! \brief The extents of the buffer. */ + Array extents; + /*! \brief Only allocate buffer when condition is satisfied. */ + PrimExpr condition; + /*! \brief The body to be executed. */ + Stmt body; + /*! \brief If the allocate is scoped global, this field indicates + * which external memories it could be pinned to as a comma seperated + * string. + */ + String pinned_memory; + ``` +TIR: + ``` +allocate_node_1 = tir.allocate([157323], "int16", "global", "foo_memory,bar_memory") + ``` + + ## S3. Using additional tags in storage_scope + +``` +/*! \brief class to represent storage scope */ +struct StorageScope { + /*! \brief The rank of the storage */ + StorageRank rank{StorageRank::kGlobal}; + /*! \brief tag for special purpose memories. */ + Array tags; +``` + + + ``` + /*! + * \brief Create storage scope from string + * \param s The string to be parsed. + * \return The storage scope. + */ + static StorageScope Create(const std::string& s) { + StorageScope r; + if (s.empty()) {/ + r.rank = StorageRank::kGlobal; + } else if (s.compare(0, 6, "global") == 0) { + r.rank = StorageRank::kGlobal; + r.tags = parseTags(s); + ``` + + +TIR (open to any other suggestion) : + +``` +allocate_node_1 = tir.allocate([157323], "int16", "global.(foo_memory,bar_memory)") +``` + + +Out of the options, S1 seems the most non-invasive. However, we will need special handlers to obtain the information. + +S2 does not change the storage scope (or the 'tags') and adds an additional field to allocates to note this information. + +S3 fold the information into storage_scope and utilizes the 'tag' denote the memory. The change to IR, is just to support more tags. + +# 5. Drawbacks + +It is a change to the IR (if we are deciding on a non-S1 solution). + +# 6. Discussion + +We'd like to hear the thoughts of the community to decide on the best way to represent the information and its string format. + + + + + + + + + From 188ade6ba8c10539d850dc849b880c7028bafa87 Mon Sep 17 00:00:00 2001 From: Manupa Karunaratne Date: Tue, 17 Aug 2021 18:06:48 +0100 Subject: [PATCH 2/7] [RFC][TIR] TIR Pinned Memory Representation * adding the PR number Change-Id: I9427b96488f73479cf96424f44fb136f9eeb07e4 --- ... => 0023-associating-allocate-nodes-with-pinned-memories.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename rfcs/{000x-associating-allocate-nodes-with-pinned-memories.md => 0023-associating-allocate-nodes-with-pinned-memories.md} (99%) diff --git a/rfcs/000x-associating-allocate-nodes-with-pinned-memories.md b/rfcs/0023-associating-allocate-nodes-with-pinned-memories.md similarity index 99% rename from rfcs/000x-associating-allocate-nodes-with-pinned-memories.md rename to rfcs/0023-associating-allocate-nodes-with-pinned-memories.md index b56a54e0..88ad1752 100644 --- a/rfcs/000x-associating-allocate-nodes-with-pinned-memories.md +++ b/rfcs/0023-associating-allocate-nodes-with-pinned-memories.md @@ -1,7 +1,7 @@ - Feature Name: TIR Pinned Memory Representation - Start Date: 2021-06-01 -- RFC PR: TBD +- RFC PR: https://github.com/apache/tvm-rfcs/pull/23 - GitHub Issue: TBD # 1. Summary From e46668b8625f6b225fa1c02cb32f4281e3ba44ed Mon Sep 17 00:00:00 2001 From: Manupa Karunaratne Date: Wed, 8 Sep 2021 16:53:04 +0100 Subject: [PATCH 3/7] [RFC][TIR] TIR Pinned Memory Representation * Changing the text to show that agreed change of addition of annotations for the allocate node to indicate the candidate memory pools while re-using the tag to indicate the committed pool that the allocate will get pinned. Change-Id: I83171137fcb17477b16891445ca10d0372b6b2e1 --- ...ing-allocate-nodes-with-pinned-memories.md | 61 +++++++++++++++++-- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/rfcs/0023-associating-allocate-nodes-with-pinned-memories.md b/rfcs/0023-associating-allocate-nodes-with-pinned-memories.md index 88ad1752..e72c9883 100644 --- a/rfcs/0023-associating-allocate-nodes-with-pinned-memories.md +++ b/rfcs/0023-associating-allocate-nodes-with-pinned-memories.md @@ -65,6 +65,58 @@ However, the intention here is to associate the name of the memory (along with o At the IR, we ll need to associate each allocate node with one (or more) memories that it can end up, because the scheduling might be satisfied with placing buffers in any of the memories in a given set of memories. Therefore, the scheduler might want the memory planners to decide which memory to use based on finding the allocation that fit. + There are broadly two requirements here : + +P1 : Indicate candidate memories (a.k.a. Pools) that each allocate be associated with + +P2 : Indicate the final memory the allocate will be pinned on + +To serve P1, we propose to use : + +``` +class AllocateNode : public StmtNode { + public: + /*! \brief The buffer variable. */ + Var buffer_var; + /*! \brief The type of the buffer. */ + DataType dtype; + /*! \brief The extents of the buffer. */ + Array extents; + /*! \brief Only allocate buffer when condition is satisfied. */ + PrimExpr condition; + /*! \brief The body to be executed. */ + Stmt body; + /*! \brief If the allocate is scoped global, this field indicates + * which external memories it could be pinned to as a comma seperated + * string. + */ ++ Map annotations; +``` + +Here the addition of the annotations field could serve offer hints/guides to future passes (i.e. Unified Static Memory Planner). We could use "candidate_memory_pools" as the key while the value being Map. + +``` +struct PoolInfoNode : public Object { + String pool_name; + Integer size_bytes; + Integer alignment; + Integer pool_offset; + Map target_access; // 'rw' or 'ro' +} +``` + +To serve P2, we propose to use the existing tag of the storage_scope with 'global-'. + +``` +struct StorageScope { + /*! \brief The rank of the storage */ + StorageRank rank{StorageRank::kGlobal}; + /*! \brief tag for special purpose memory. */ + std::string tag; +``` + +# Alternatives + ## S1. Using AttrStmt to associate additional info : TIR: @@ -127,7 +179,7 @@ struct StorageScope { ``` -TIR (open to any other suggestion) : +TIR: ``` allocate_node_1 = tir.allocate([157323], "int16", "global.(foo_memory,bar_memory)") @@ -140,13 +192,12 @@ S2 does not change the storage scope (or the 'tags') and adds an additional fiel S3 fold the information into storage_scope and utilizes the 'tag' denote the memory. The change to IR, is just to support more tags. -# 5. Drawbacks +However, in the discussion with community, we felt the need to seperate changes that serves P1 and P2, respectively. In fact, for P2 we could re-use the tag of storage_scope without an IR change. For P1, it seems a good and general change to include annotations for the allocate node. -It is a change to the IR (if we are deciding on a non-S1 solution). +# 5. Drawbacks -# 6. Discussion +It is not direct because we are using general annotations to indicate this, however it is consistent with rest of the IR and allows future expansions. -We'd like to hear the thoughts of the community to decide on the best way to represent the information and its string format. From c447cbfbd5abceaa7623a0f90cc492784e6f0c0b Mon Sep 17 00:00:00 2001 From: Manupa Karunaratne Date: Mon, 20 Sep 2021 07:24:44 +0100 Subject: [PATCH 4/7] [RFC][TIR] adding annotation field to tir.allocate Moved the pinned memory representation to adding generic annotation field, after the community discussion. Change-Id: I122cb0bd9d72bd2350003908d14e187d0902cbff --- ...dding-annotation-field-to-tir.allocate.md} | 58 ++++++++----------- 1 file changed, 25 insertions(+), 33 deletions(-) rename rfcs/{0023-associating-allocate-nodes-with-pinned-memories.md => 0023-adding-annotation-field-to-tir.allocate.md} (79%) diff --git a/rfcs/0023-associating-allocate-nodes-with-pinned-memories.md b/rfcs/0023-adding-annotation-field-to-tir.allocate.md similarity index 79% rename from rfcs/0023-associating-allocate-nodes-with-pinned-memories.md rename to rfcs/0023-adding-annotation-field-to-tir.allocate.md index e72c9883..c56b9338 100644 --- a/rfcs/0023-associating-allocate-nodes-with-pinned-memories.md +++ b/rfcs/0023-adding-annotation-field-to-tir.allocate.md @@ -1,14 +1,14 @@ -- Feature Name: TIR Pinned Memory Representation +- Feature Name: Adding annotatation field to tir.allocate nodes - Start Date: 2021-06-01 - RFC PR: https://github.com/apache/tvm-rfcs/pull/23 - GitHub Issue: TBD # 1. Summary -This RFC proposes how pinned memories could be associated with tir.allocate nodes (and allocate_const nodes) and used by passes in the lowering process. +This RFC proposes to annotation field tir.allocate nodes. These annotations can be used as auxiliary hint to future transformations. -# 2. Motivation +# 2. Motivational usecase : pinned memory Currently, TVM relies on dynamic (alloc and free style) allocations in runtime to manage the intermediary memory used by operators and the network. This is sometimes not desirable, especially in microTVM. @@ -53,23 +53,34 @@ The current design of [Unified Static Memory Planner (USMP)](https://github.com/ Therefore, we'd need a way to represent the association of each of these memories, that the user will pin the buffers to, closer to allocate nodes in TIR. -# 3. Guide-level explanation +At the IR, we ll need to associate each allocate node with one (or more) memories that it can end up, because the scheduling might be satisfied with placing buffers in any of the memories in a given set of memories. Therefore, the scheduler might want the memory planners to decide which memory to use based on finding the allocation that fit. -This is not particularly a user-facing feature. + There are broadly two requirements here : -However, the intention here is to associate the name of the memory (along with other information) closer to the allocate IR node. Therefore at the end of the compilation, metadata module (+ header) will generate pointer structs to-be passed in the from the application layer. +P1 : Indicate candidate memories (a.k.a. Pools) that each allocate be associated with - In the case where user does not wish to pass in workspace or constant buffers, metadata module will generate a workspace buffer and a constant buffer, as explained in U1 of [USMP](https://github.com/apache/tvm-rfcs/pull/9). +P2 : Indicate the final memory the allocate will be pinned on - # 4. Reference-level explanation - At the IR, we ll need to associate each allocate node with one (or more) memories that it can end up, because the scheduling might be satisfied with placing buffers in any of the memories in a given set of memories. Therefore, the scheduler might want the memory planners to decide which memory to use based on finding the allocation that fit. +To serve P2, we propose to use the existing tag of the storage_scope with 'global-'. - There are broadly two requirements here : +``` +struct StorageScope { + /*! \brief The rank of the storage */ + StorageRank rank{StorageRank::kGlobal}; + /*! \brief tag for special purpose memory. */ + std::string tag; +``` -P1 : Indicate candidate memories (a.k.a. Pools) that each allocate be associated with +To serve P1, this RFC introduces the addition of the annotations field + +# 3. Guide-level explanation + +This is not particularly a user-facing feature. + + + # 4. Reference-level explanation -P2 : Indicate the final memory the allocate will be pinned on To serve P1, we propose to use : @@ -93,27 +104,8 @@ class AllocateNode : public StmtNode { + Map annotations; ``` -Here the addition of the annotations field could serve offer hints/guides to future passes (i.e. Unified Static Memory Planner). We could use "candidate_memory_pools" as the key while the value being Map. - -``` -struct PoolInfoNode : public Object { - String pool_name; - Integer size_bytes; - Integer alignment; - Integer pool_offset; - Map target_access; // 'rw' or 'ro' -} -``` - -To serve P2, we propose to use the existing tag of the storage_scope with 'global-'. +Here the addition of the annotations field could serve offer hints/guides to future passes (i.e. In Unified Static Memory Planner, we could use "candidate_memory_pools" as the key while the value being Map.) -``` -struct StorageScope { - /*! \brief The rank of the storage */ - StorageRank rank{StorageRank::kGlobal}; - /*! \brief tag for special purpose memory. */ - std::string tag; -``` # Alternatives @@ -196,7 +188,7 @@ However, in the discussion with community, we felt the need to seperate changes # 5. Drawbacks -It is not direct because we are using general annotations to indicate this, however it is consistent with rest of the IR and allows future expansions. +None. Its consistent with rest of the IR design and allows other features to use to pass hints for future transformation in the compiler. From 56ea902f33f8995db9609758c1a069c7810f9327 Mon Sep 17 00:00:00 2001 From: Manupa Karunaratne Date: Wed, 22 Sep 2021 11:12:24 +0100 Subject: [PATCH 5/7] [RFC][TIR] TIR Pinned Memory Representation * addressing Andrew's comments Change-Id: Ic9a4c6c0cff43057e5f2e7f9654355fe7c91ed24 --- ...adding-annotation-field-to-tir.allocate.md | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/rfcs/0023-adding-annotation-field-to-tir.allocate.md b/rfcs/0023-adding-annotation-field-to-tir.allocate.md index c56b9338..a4961af0 100644 --- a/rfcs/0023-adding-annotation-field-to-tir.allocate.md +++ b/rfcs/0023-adding-annotation-field-to-tir.allocate.md @@ -51,26 +51,18 @@ The current design of [Unified Static Memory Planner (USMP)](https://github.com/ } ``` -Therefore, we'd need a way to represent the association of each of these memories, that the user will pin the buffers to, closer to allocate nodes in TIR. +Therefore, we'd need a way to represent the association of each of these memories, that the user will pin the buffers (e.g. workspace_buffer_2 to SRAM) to. That infomation should flow closer to allocate nodes in TIR in the compilation passes. -At the IR, we ll need to associate each allocate node with one (or more) memories that it can end up, because the scheduling might be satisfied with placing buffers in any of the memories in a given set of memories. Therefore, the scheduler might want the memory planners to decide which memory to use based on finding the allocation that fit. +At the IR, we ll need to associate each allocate node with one (or more) memory pools that it can end up, because the scheduling might be satisfied with placing buffers in any of the memory pools in a given set of memory pools. Therefore, the scheduler might want the memory planners to decide which memory pool to use based on finding the allocation that fit. There are broadly two requirements here : -P1 : Indicate candidate memories (a.k.a. Pools) that each allocate be associated with +P1 : Indicate candidate memory pools (a.k.a. PoolInfo Objects -- for further details see [USMP RFC](https://github.com/apache/tvm-rfcs/pull/9)) that each allocate be associated with -P2 : Indicate the final memory the allocate will be pinned on +P2 : Indicate the final memory pool the allocate will be pinned on -To serve P2, we propose to use the existing tag of the storage_scope with 'global-'. - -``` -struct StorageScope { - /*! \brief The rank of the storage */ - StorageRank rank{StorageRank::kGlobal}; - /*! \brief tag for special purpose memory. */ - std::string tag; -``` +To serve P2, we are proposing to use the existing tag of the storage_scope with 'global.' in the [USMP RFC](https://github.com/apache/tvm-rfcs/pull/9). To serve P1, this RFC introduces the addition of the annotations field @@ -78,6 +70,8 @@ To serve P1, this RFC introduces the addition of the annotations field This is not particularly a user-facing feature. +The proposal in this RFC is to add an annotation field to tir.allocate node to be used by compilation passes. + # 4. Reference-level explanation From e6a52711022e9554d405014b4815b83fb3cc5131 Mon Sep 17 00:00:00 2001 From: Manupa Karunaratne Date: Thu, 30 Sep 2021 15:41:27 +0100 Subject: [PATCH 6/7] [RFC][TIR] TIR Pinned Memory Representation Making the brief for annotation generic for tir.allocate --- rfcs/0023-adding-annotation-field-to-tir.allocate.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rfcs/0023-adding-annotation-field-to-tir.allocate.md b/rfcs/0023-adding-annotation-field-to-tir.allocate.md index a4961af0..654efc15 100644 --- a/rfcs/0023-adding-annotation-field-to-tir.allocate.md +++ b/rfcs/0023-adding-annotation-field-to-tir.allocate.md @@ -91,9 +91,13 @@ class AllocateNode : public StmtNode { PrimExpr condition; /*! \brief The body to be executed. */ Stmt body; - /*! \brief If the allocate is scoped global, this field indicates - * which external memories it could be pinned to as a comma seperated - * string. + /*! + * \brief Additional annotations about the loop. + * + * These annotations can be used as auxiliary hint + * to future transformations. An annotation should + * not change the control flow semantics of the loop + * and can be ignored in most passes. */ + Map annotations; ``` From 0863fc174a1892101a1ce33c152fb471f78bbe60 Mon Sep 17 00:00:00 2001 From: Manupa Karunaratne Date: Thu, 30 Sep 2021 20:14:53 +0100 Subject: [PATCH 7/7] [RFC][TIR] TIR Pinned Memory Representation * type in the brief in the example --- rfcs/0023-adding-annotation-field-to-tir.allocate.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rfcs/0023-adding-annotation-field-to-tir.allocate.md b/rfcs/0023-adding-annotation-field-to-tir.allocate.md index 654efc15..61b97421 100644 --- a/rfcs/0023-adding-annotation-field-to-tir.allocate.md +++ b/rfcs/0023-adding-annotation-field-to-tir.allocate.md @@ -95,9 +95,7 @@ class AllocateNode : public StmtNode { * \brief Additional annotations about the loop. * * These annotations can be used as auxiliary hint - * to future transformations. An annotation should - * not change the control flow semantics of the loop - * and can be ignored in most passes. + * to future transformations. */ + Map annotations; ```