diff --git a/be/src/vec/common/pod_array.h b/be/src/vec/common/pod_array.h index 959dda59cacf9d..8179338548f961 100644 --- a/be/src/vec/common/pod_array.h +++ b/be/src/vec/common/pod_array.h @@ -73,6 +73,9 @@ namespace doris::vectorized { * and we must avoid larger object size due to storing the same parameters in each object. * This is required for states of aggregate functions. * + * PODArray does not have memset 0 when allocating memory, therefore, the query mem tracker is virtual memory, + * which will cause the query memory statistics to be higher than the actual physical memory. + * * TODO Pass alignment to Allocator. * TODO Allow greater alignment than alignof(T). Example: array of char aligned to page size. */ @@ -102,7 +105,6 @@ class PODArrayBase : private boost::noncopyable, char* c_start = null; /// Does not include pad_left. char* c_end = null; char* c_end_of_storage = null; /// Does not include pad_right. - char* c_end_peak = null; /// The amount of memory occupied by the num_elements of the elements. static size_t byte_size(size_t num_elements) { return num_elements * ELEMENT_SIZE; } @@ -112,26 +114,15 @@ class PODArrayBase : private boost::noncopyable, return byte_size(num_elements) + pad_right + pad_left; } - inline void reset_peak() { - if (UNLIKELY(c_end - c_end_peak > 65536)) { - THREAD_MEM_TRACKER_TRANSFER_FROM(c_end - c_end_peak, - ExecEnv::GetInstance()->orphan_mem_tracker_raw()); - c_end_peak = c_end; - } - } - void alloc_for_num_elements(size_t num_elements) { alloc(round_up_to_power_of_two_or_zero(minimum_memory_for_elements(num_elements))); } template void alloc(size_t bytes, TAllocatorParams&&... allocator_params) { - THREAD_MEM_TRACKER_TRANSFER_TO(bytes - pad_right - pad_left, - ExecEnv::GetInstance()->orphan_mem_tracker_raw()); - c_start = c_end = c_end_peak = - reinterpret_cast(TAllocator::alloc( - bytes, std::forward(allocator_params)...)) + - pad_left; + c_start = c_end = reinterpret_cast(TAllocator::alloc( + bytes, std::forward(allocator_params)...)) + + pad_left; c_end_of_storage = c_start + bytes - pad_right - pad_left; if (pad_left) memset(c_start - ELEMENT_SIZE, 0, ELEMENT_SIZE); @@ -143,8 +134,6 @@ class PODArrayBase : private boost::noncopyable, unprotect(); TAllocator::free(c_start - pad_left, allocated_bytes()); - THREAD_MEM_TRACKER_TRANSFER_FROM(c_end_of_storage - c_end_peak, - ExecEnv::GetInstance()->orphan_mem_tracker_raw()); } template @@ -156,11 +145,7 @@ class PODArrayBase : private boost::noncopyable, unprotect(); - THREAD_MEM_TRACKER_TRANSFER_TO(bytes - allocated_bytes(), - ExecEnv::GetInstance()->orphan_mem_tracker_raw()); - ptrdiff_t end_diff = c_end - c_start; - ptrdiff_t peak_diff = c_end_peak - c_start; c_start = reinterpret_cast(TAllocator::realloc( c_start - pad_left, allocated_bytes(), bytes, @@ -168,7 +153,6 @@ class PODArrayBase : private boost::noncopyable, pad_left; c_end = c_start + end_diff; - c_end_peak = c_start + peak_diff; c_end_of_storage = c_start + bytes - pad_right - pad_left; } @@ -239,10 +223,7 @@ class PODArrayBase : private boost::noncopyable, resize_assume_reserved(n); } - void resize_assume_reserved(const size_t n) { - c_end = c_start + byte_size(n); - reset_peak(); - } + void resize_assume_reserved(const size_t n) { c_end = c_start + byte_size(n); } const char* raw_data() const { return c_start; } @@ -253,7 +234,6 @@ class PODArrayBase : private boost::noncopyable, memcpy(c_end, ptr, ELEMENT_SIZE); c_end += byte_size(1); - reset_peak(); } void protect() { @@ -306,7 +286,6 @@ class PODArray : public PODArrayBasealloc_for_num_elements(n); this->c_end += this->byte_size(n); - this->reset_peak(); } PODArray(size_t n, const T& x) { @@ -358,10 +337,7 @@ class PODArray : public PODArrayBasec_end; } - void set_end_ptr(void* ptr) { - this->c_end = (char*)ptr; - this->reset_peak(); - } + void set_end_ptr(void* ptr) { this->c_end = (char*)ptr; } /// Same as resize, but zeroes new elements. void resize_fill(size_t n) { @@ -371,7 +347,6 @@ class PODArray : public PODArrayBasec_end, 0, this->byte_size(n - old_size)); } this->c_end = this->c_start + this->byte_size(n); - this->reset_peak(); } void resize_fill(size_t n, const T& value) { @@ -381,7 +356,6 @@ class PODArray : public PODArrayBasec_end = this->c_start + this->byte_size(n); - this->reset_peak(); } template @@ -391,7 +365,6 @@ class PODArray : public PODArrayBase(x)); this->c_end += this->byte_size(1); - this->reset_peak(); } template @@ -403,7 +376,6 @@ class PODArray : public PODArrayBasec_end = new_end; - this->reset_peak(); } } @@ -412,7 +384,6 @@ class PODArray : public PODArrayBasec_end += sizeof(T) * num; - this->reset_peak(); } /** @@ -423,7 +394,6 @@ class PODArray : public PODArrayBase(x)); this->c_end += this->byte_size(1); - this->reset_peak(); } /** This method doesn't allow to pass parameters for Allocator, @@ -435,7 +405,6 @@ class PODArray : public PODArrayBase(args)...); this->c_end += this->byte_size(1); - this->reset_peak(); } void pop_back() { this->c_end -= this->byte_size(1); } @@ -479,7 +448,6 @@ class PODArray : public PODArrayBasec_end, reinterpret_cast(&*from_begin), bytes_to_copy); this->c_end += bytes_to_copy; - this->reset_peak(); } template @@ -496,7 +464,6 @@ class PODArray : public PODArrayBasec_end - bytes_to_move, reinterpret_cast(&*from_begin), bytes_to_copy); this->c_end += bytes_to_copy; - this->reset_peak(); } template @@ -504,7 +471,6 @@ class PODArray : public PODArrayBasebyte_size(from_end - from_begin); memcpy(this->c_end, reinterpret_cast(&*from_begin), bytes_to_copy); this->c_end += bytes_to_copy; - this->reset_peak(); } void swap(PODArray& rhs) { @@ -523,11 +489,6 @@ class PODArray : public PODArrayBasebyte_size(heap_size); - arr1.c_end_peak = arr1.c_end; - THREAD_MEM_TRACKER_TRANSFER_FROM(arr1.c_end_peak - arr1.c_start - heap_peak_used, - ExecEnv::GetInstance()->orphan_mem_tracker_raw()); /// Allocate stack space for arr2. arr2.alloc(stack_allocated); /// Copy the stack content. memcpy(arr2.c_start, stack_c_start, this->byte_size(stack_size)); - arr2.c_end = arr2.c_end_peak = arr2.c_start + this->byte_size(stack_size); - THREAD_MEM_TRACKER_TRANSFER_FROM(arr2.c_end_peak - arr2.c_start - stack_peak_used, - ExecEnv::GetInstance()->orphan_mem_tracker_raw()); + arr2.c_end = arr2.c_start + this->byte_size(stack_size); }; auto do_move = [this](PODArray& src, PODArray& dest) { @@ -553,21 +509,15 @@ class PODArray : public PODArrayBasebyte_size(src.size())); - dest.c_end = dest.c_end_peak = dest.c_start + (src.c_end - src.c_start); - THREAD_MEM_TRACKER_TRANSFER_FROM(dest.c_end_peak - dest.c_start, - ExecEnv::GetInstance()->orphan_mem_tracker_raw()); + dest.c_end = dest.c_start + (src.c_end - src.c_start); - THREAD_MEM_TRACKER_TRANSFER_FROM(src.c_end_of_storage - src.c_end_peak, - ExecEnv::GetInstance()->orphan_mem_tracker_raw()); src.c_start = Base::null; src.c_end = Base::null; src.c_end_of_storage = Base::null; - src.c_end_peak = Base::null; } else { std::swap(dest.c_start, src.c_start); std::swap(dest.c_end, src.c_end); std::swap(dest.c_end_of_storage, src.c_end_of_storage); - std::swap(dest.c_end_peak, src.c_end_peak); } }; @@ -604,8 +554,6 @@ class PODArray : public PODArrayBasec_end = this->c_start + this->byte_size(rhs_size); rhs.c_end = rhs.c_start + this->byte_size(lhs_size); - this->reset_peak(); - rhs.reset_peak(); } else if (this->is_allocated_from_stack() && !rhs.is_allocated_from_stack()) { swap_stack_heap(*this, rhs); } else if (!this->is_allocated_from_stack() && rhs.is_allocated_from_stack()) { @@ -614,7 +562,6 @@ class PODArray : public PODArrayBasec_start, rhs.c_start); std::swap(this->c_end, rhs.c_end); std::swap(this->c_end_of_storage, rhs.c_end_of_storage); - std::swap(this->c_end_peak, rhs.c_end_peak); } } @@ -632,7 +579,6 @@ class PODArray : public PODArrayBasebyte_size(required_capacity); memcpy(this->c_start, reinterpret_cast(&*from_begin), bytes_to_copy); this->c_end = this->c_start + bytes_to_copy; - this->reset_peak(); } void assign(const PODArray& from) { assign(from.begin(), from.end()); }