diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index bf177742ab3563..7d3f4da935099e 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -112,23 +112,24 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu
     if (_closed) {
         return Status::OK();
     }
-    auto p = _parent->cast<HashJoinBuildSinkOperatorX>();
     Defer defer {[&]() {
-        if (_should_build_hash_table) {
-            // The build side hash key column maybe no need output, but we need to keep the column in block
-            // because it is used to compare with probe side hash key column
-            if (p._should_keep_hash_key_column && _build_col_ids.size() == 1) {
-                p._should_keep_column_flags[_build_col_ids[0]] = true;
-            }
+        if (!_should_build_hash_table) {
+            return;
+        }
+        // The build side hash key column maybe no need output, but we need to keep the column in block
+        // because it is used to compare with probe side hash key column
+        auto p = _parent->cast<HashJoinBuildSinkOperatorX>();
+        if (p._should_keep_hash_key_column && _build_col_ids.size() == 1) {
+            p._should_keep_column_flags[_build_col_ids[0]] = true;
+        }
 
-            if (_shared_state->build_block) {
-                // release the memory of unused column in probe stage
-                _shared_state->build_block->clear_column_mem_not_keep(
-                        p._should_keep_column_flags, bool(p._shared_hashtable_controller));
-            }
+        if (_shared_state->build_block) {
+            // release the memory of unused column in probe stage
+            _shared_state->build_block->clear_column_mem_not_keep(
+                    p._should_keep_column_flags, bool(p._shared_hashtable_controller));
         }
 
-        if (_should_build_hash_table && p._shared_hashtable_controller) {
+        if (p._shared_hashtable_controller) {
             p._shared_hashtable_controller->signal_finish(p.node_id());
         }
     }};
@@ -137,22 +138,22 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu
         return Base::close(state, exec_status);
     }
 
-    if (state->get_task()->wake_up_by_downstream()) {
-        RETURN_IF_ERROR(_runtime_filter_slots->send_filter_size(state, 0, _finish_dependency));
-        RETURN_IF_ERROR(_runtime_filter_slots->ignore_all_filters());
-    } else {
-        auto* block = _shared_state->build_block.get();
-        uint64_t hash_table_size = block ? block->rows() : 0;
-        {
-            SCOPED_TIMER(_runtime_filter_init_timer);
-            if (_should_build_hash_table) {
+    if (_should_build_hash_table) {
+        if (state->get_task()->wake_up_by_downstream()) {
+            RETURN_IF_ERROR(_runtime_filter_slots->send_filter_size(state, 0, _finish_dependency));
+            RETURN_IF_ERROR(_runtime_filter_slots->ignore_all_filters());
+        } else {
+            auto* block = _shared_state->build_block.get();
+            uint64_t hash_table_size = block ? block->rows() : 0;
+            {
+                SCOPED_TIMER(_runtime_filter_init_timer);
                 RETURN_IF_ERROR(_runtime_filter_slots->init_filters(state, hash_table_size));
+                RETURN_IF_ERROR(_runtime_filter_slots->ignore_filters(state));
+            }
+            if (hash_table_size > 1) {
+                SCOPED_TIMER(_runtime_filter_compute_timer);
+                _runtime_filter_slots->insert(block);
             }
-            RETURN_IF_ERROR(_runtime_filter_slots->ignore_filters(state));
-        }
-        if (_should_build_hash_table && hash_table_size > 1) {
-            SCOPED_TIMER(_runtime_filter_compute_timer);
-            _runtime_filter_slots->insert(block);
         }
     }
     SCOPED_TIMER(_publish_runtime_filter_timer);