From 59c369770724ec4d7ed161bc70038b9e87c91776 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 4 Apr 2023 08:09:10 +0000 Subject: [PATCH 1/3] [pre-commit.ci] pre-commit suggestions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/hadialqattan/pycln: v2.1.2 → v2.1.3](https://github.com/hadialqattan/pycln/compare/v2.1.2...v2.1.3) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1fe4f18ada..e0a4e84eb4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,7 +64,7 @@ repos: )$ - repo: https://github.com/hadialqattan/pycln - rev: v2.1.2 + rev: v2.1.3 hooks: - id: pycln args: [--config=pyproject.toml] From e417fd52a02e0502c4857c544ec085f54419e4fa Mon Sep 17 00:00:00 2001 From: Wenqi Li <831580+wyli@users.noreply.github.com> Date: Tue, 4 Apr 2023 11:47:36 +0100 Subject: [PATCH 2/3] Revert "feat(SABlock): access to the attn matrix" (#6288) Reverts Project-MONAI/MONAI#6271 --- monai/networks/blocks/selfattention.py | 23 +++++------------------ monai/networks/blocks/transformerblock.py | 13 +++---------- tests/test_selfattention.py | 20 -------------------- 3 files changed, 8 insertions(+), 48 deletions(-) diff --git a/monai/networks/blocks/selfattention.py b/monai/networks/blocks/selfattention.py index 03a7bc7e08..519c8c7728 100644 --- a/monai/networks/blocks/selfattention.py +++ b/monai/networks/blocks/selfattention.py @@ -25,21 +25,13 @@ class SABlock(nn.Module): An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale " """ - def __init__( - self, - hidden_size: int, - num_heads: int, - dropout_rate: float = 0.0, - qkv_bias: bool = False, - save_attn: bool = False, - ) -> None: + def __init__(self, hidden_size: int, num_heads: int, dropout_rate: float = 0.0, qkv_bias: bool = False) -> None: """ Args: - hidden_size (int): dimension of hidden layer. - num_heads (int): number of attention heads. - dropout_rate (float, optional): faction of the input units to drop. Defaults to 0.0. - qkv_bias (bool, optional): bias term for the qkv linear layer. Defaults to False. - save_attn (bool, optional): to make accessible the attention matrix. Defaults to False. + hidden_size: dimension of hidden layer. + num_heads: number of attention heads. + dropout_rate: faction of the input units to drop. + qkv_bias: bias term for the qkv linear layer. """ @@ -60,16 +52,11 @@ def __init__( self.drop_weights = nn.Dropout(dropout_rate) self.head_dim = hidden_size // num_heads self.scale = self.head_dim**-0.5 - self.save_attn = save_attn def forward(self, x): output = self.input_rearrange(self.qkv(x)) q, k, v = output[0], output[1], output[2] att_mat = (torch.einsum("blxd,blyd->blxy", q, k) * self.scale).softmax(dim=-1) - if self.save_attn: - # no gradients and new tensor; - # https://pytorch.org/docs/stable/generated/torch.Tensor.detach.html - self.att_mat = att_mat.detach() att_mat = self.drop_weights(att_mat) x = torch.einsum("bhxy,bhyd->bhxd", att_mat, v) x = self.out_rearrange(x) diff --git a/monai/networks/blocks/transformerblock.py b/monai/networks/blocks/transformerblock.py index 30f2c2756a..3a4b507d69 100644 --- a/monai/networks/blocks/transformerblock.py +++ b/monai/networks/blocks/transformerblock.py @@ -24,13 +24,7 @@ class TransformerBlock(nn.Module): """ def __init__( - self, - hidden_size: int, - mlp_dim: int, - num_heads: int, - dropout_rate: float = 0.0, - qkv_bias: bool = False, - save_attn: bool = False, + self, hidden_size: int, mlp_dim: int, num_heads: int, dropout_rate: float = 0.0, qkv_bias: bool = False ) -> None: """ Args: @@ -38,8 +32,7 @@ def __init__( mlp_dim: dimension of feedforward layer. num_heads: number of attention heads. dropout_rate: faction of the input units to drop. - qkv_bias: apply bias term for the qkv linear layer. - save_attn: to make accessible the attention matrix post training. + qkv_bias: apply bias term for the qkv linear layer """ @@ -53,7 +46,7 @@ def __init__( self.mlp = MLPBlock(hidden_size, mlp_dim, dropout_rate) self.norm1 = nn.LayerNorm(hidden_size) - self.attn = SABlock(hidden_size, num_heads, dropout_rate, qkv_bias, save_attn) + self.attn = SABlock(hidden_size, num_heads, dropout_rate, qkv_bias) self.norm2 = nn.LayerNorm(hidden_size) def forward(self, x): diff --git a/tests/test_selfattention.py b/tests/test_selfattention.py index a67f54f704..926ef7da55 100644 --- a/tests/test_selfattention.py +++ b/tests/test_selfattention.py @@ -52,26 +52,6 @@ def test_ill_arg(self): with self.assertRaises(ValueError): SABlock(hidden_size=620, num_heads=8, dropout_rate=0.4) - def test_access_attn_matrix(self): - # input format - hidden_size = 128 - num_heads = 2 - dropout_rate = 0 - input_shape = (2, 256, hidden_size) - - # be able to access the matrix - no_matrix_acess_blk = SABlock(hidden_size=hidden_size, num_heads=num_heads, dropout_rate=dropout_rate) - with self.assertRaises(AttributeError): - no_matrix_acess_blk(torch.randn(input_shape)) - no_matrix_acess_blk.att_mat - - # be not able to acess the attention matrix - matrix_acess_blk = SABlock( - hidden_size=hidden_size, num_heads=num_heads, dropout_rate=dropout_rate, save_attn=True - ) - matrix_acess_blk(torch.randn(input_shape)) - assert matrix_acess_blk.att_mat.shape == (input_shape[0], input_shape[0], input_shape[1], input_shape[1]) - if __name__ == "__main__": unittest.main() From c91d79af3b5d3896d09e4bf9e529cef1dc5522a0 Mon Sep 17 00:00:00 2001 From: Mingxin Zheng <18563433+mingxin-zheng@users.noreply.github.com> Date: Tue, 4 Apr 2023 06:50:35 -0400 Subject: [PATCH 3/3] Fix Auto3DSeg DataAnalyzer test "cpuonly" is not using CPU (#6278) Fixes #6277 . ### Description A few sentences describing the changes proposed in this pull request. ### Types of changes - [x] Non-breaking change (fix or new feature that would not break existing functionality). --------- Signed-off-by: Mingxin Zheng <18563433+mingxin-zheng@users.noreply.github.com> --- tests/test_auto3dseg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_auto3dseg.py b/tests/test_auto3dseg.py index deb54b7a4b..53f25051ec 100644 --- a/tests/test_auto3dseg.py +++ b/tests/test_auto3dseg.py @@ -184,7 +184,7 @@ def test_data_analyzer_cpu(self, input_params): ) analyser = DataAnalyzer( - self.datalist_file, self.dataroot_dir, output_path=self.datastat_file, label_key=label_key + self.datalist_file, self.dataroot_dir, output_path=self.datastat_file, label_key=label_key, device=device ) datastat = analyser.get_all_case_stats()