diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp
index a30a4fe3ddee03..400da13436695f 100644
--- a/be/src/olap/delta_writer.cpp
+++ b/be/src/olap/delta_writer.cpp
@@ -38,6 +38,9 @@ DeltaWriter::~DeltaWriter() {
     if (!_delta_written_success) {
         _garbage_collection();
     }
+    for (SegmentGroup* segment_group : _segment_group_vec) {
+        segment_group->release();
+    }
     SAFE_DELETE(_writer);
     SAFE_DELETE(_mem_table);
     SAFE_DELETE(_schema);
@@ -47,14 +50,12 @@ void DeltaWriter::_garbage_collection() {
     OLAPEngine::get_instance()->delete_transaction(_req.partition_id, _req.transaction_id,
                                                    _req.tablet_id, _req.schema_hash);
     for (SegmentGroup* segment_group : _segment_group_vec) {
-        segment_group->release();
         OLAPEngine::get_instance()->add_unused_index(segment_group);
     }
     if (_new_table != nullptr) {
         OLAPEngine::get_instance()->delete_transaction(_req.partition_id, _req.transaction_id,
                                                        _new_table->tablet_id(), _new_table->schema_hash());
         for (SegmentGroup* segment_group : _new_segment_group_vec) {
-            segment_group->release();
             OLAPEngine::get_instance()->add_unused_index(segment_group);
         }
     }