Reworked FramePassTimer to prevent stalling the GPU

include/RenderGraph/FramePassTimer.hpp

@@ -7,6 +7,7 @@ See LICENSE file in root folder
 #include "FrameGraphPrerequisites.hpp"
 #include "Signal.hpp"
 
+#include <array>
 #include <chrono>
 
 namespace crg
@@ -50,14 +51,11 @@ namespace crg
 		*	The render pass category.
 		*\param[in] name
 		*	The timer name.
-		*\param[in] passesCount
-		*	The number of render passes.
 		*/
 		CRG_API FramePassTimer( GraphContext & context
 			, std::string const & name
 			, VkQueryPool timerQueries
-			, uint32_t & baseQueryOffset
-			, uint32_t passesCount = 1u );
+			, uint32_t & baseQueryOffset );
 		/**
 		*\brief
 		*	Owns its query pool.
@@ -67,12 +65,9 @@ namespace crg
 		*	The render pass category.
 		*\param[in] name
 		*	The timer name.
-		*\param[in] passesCount
-		*	The number of render passes.
 		*/
 		CRG_API FramePassTimer( GraphContext & context
-			, std::string const & name
-			, uint32_t passesCount = 1u );
+			, std::string const & name );
 		CRG_API ~FramePassTimer();
 		/**
 		*\brief
@@ -102,8 +97,7 @@ namespace crg
 		*\param[in] passIndex
 		*	The pass index.
 		*/
-		CRG_API void beginPass( VkCommandBuffer commandBuffer
-			, uint32_t passIndex = 0u )const;
+		CRG_API void beginPass( VkCommandBuffer commandBuffer );
 		/**
 		*\brief
 		*	Writes the timestamp for the end of the pass.
@@ -112,26 +106,13 @@ namespace crg
 		*\param[in] passIndex
 		*	The pass index.
 		*/
-		CRG_API void endPass( VkCommandBuffer commandBuffer
-			, uint32_t passIndex = 0u )const;
+		CRG_API void endPass( VkCommandBuffer commandBuffer );
 		/**
 		*\brief
 		*	Retrieves GPU time from the query.
 		*/
 		CRG_API void retrieveGpuTime();
 		/**
-		*\brief
-		*	Updates the passes count to the given value.
-		*\param[in] count
-		*	The number of render passes.
-		*\~french
-		*\brief
-		*	Met à jour le nombre de passes à la valeur donnée.
-		*\param[in] count
-		*	Le nombre de passes de rendu.
-		*/
-		CRG_API void updateCount( uint32_t count );
-		/**
 		*\name
 		*	Getters.
 		*/
@@ -146,11 +127,6 @@ namespace crg
 			return m_gpuTime;
 		}
 
-		uint32_t getCount()const
-		{
-			return m_passesCount;
-		}
-
 		std::string const & getName()const
 		{
 			return m_name;
@@ -164,16 +140,21 @@ namespace crg
 
 	private:
 		GraphContext & m_context;
-		uint32_t m_passesCount;
 		std::string m_name;
 		Clock::time_point m_cpuSaveTime;
 		Nanoseconds m_cpuTime;
 		Nanoseconds m_gpuTime;
-		Nanoseconds m_subtracteGpuTime;
+		Nanoseconds m_subtractedGpuTime;
 		VkQueryPool m_timerQueries;
-		uint32_t m_baseQueryOffset{};
 		bool m_ownPool{};
-		std::vector< std::pair< bool, bool > > m_startedPasses;
+		struct Query
+		{
+			uint32_t offset{};
+			bool written{};
+			bool started{};
+			bool subtractGpuFromCpu{};
+		};
+		std::array< Query, 2u > m_queries;
 	};
 }
 

include/RenderGraph/RunnablePass.hpp

@@ -261,6 +261,7 @@ namespace crg
 		*/
 		CRG_API void resetCommandBuffer( uint32_t passIndex );
 		CRG_API void setToReset( uint32_t passIndex );
+		CRG_API void notifyPassRender();
 
 		LayoutState getLayoutState( crg::ImageViewId view )const
 		{

source/RenderGraph/FramePassTimer.cpp

@@ -44,35 +44,29 @@ namespace crg
 	FramePassTimer::FramePassTimer( GraphContext & context
 		, std::string const & name
 		, VkQueryPool timerQueries
-		, uint32_t & baseQueryOffset
-		, uint32_t passesCount )
+		, uint32_t & baseQueryOffset )
 		: m_context{ context }
-		, m_passesCount{ passesCount }
 		, m_name{ name }
 		, m_cpuTime{ 0ns }
 		, m_gpuTime{ 0ns }
 		, m_timerQueries{ timerQueries }
-		, m_baseQueryOffset{ baseQueryOffset }
 		, m_ownPool{ false }
-		, m_startedPasses( size_t( m_passesCount ), { false, false } )
+		, m_queries{ { { baseQueryOffset, false, false }, { baseQueryOffset + 2u, false, false } } }
 	{
-		baseQueryOffset += passesCount * 2u;
+		baseQueryOffset += 4u;
 	}
 
 	FramePassTimer::FramePassTimer( GraphContext & context
-		, std::string const & name
-		, uint32_t passesCount )
+		, std::string const & name )
 		: m_context{ context }
-		, m_passesCount{ passesCount }
 		, m_name{ name }
 		, m_cpuTime{ 0ns }
 		, m_gpuTime{ 0ns }
 		, m_timerQueries{ createQueryPool( context
 			, name
-			, passesCount * 2u ) }
-		, m_baseQueryOffset{ 0u }
+			, 4u ) }
 		, m_ownPool{ true }
-		, m_startedPasses( size_t( m_passesCount ), { false, false } )
+		, m_queries{ { { 0u, false, false }, { 2u, false, false } } }
 	{
 	}
 
@@ -98,16 +92,16 @@ namespace crg
 	void FramePassTimer::notifyPassRender( uint32_t passIndex
 		, bool subtractGpuFromCpu )
 	{
-		auto & started = m_startedPasses[passIndex];
-		started.first = true;
-		started.second = subtractGpuFromCpu;
+		auto & query = m_queries.front();
+		query.started = true;
+		query.subtractGpuFromCpu = subtractGpuFromCpu;
 	}
 
 	void FramePassTimer::stop()
 	{
 		auto current = Clock::now();
 		m_cpuTime += ( current - m_cpuSaveTime );
-		m_cpuTime -= m_subtracteGpuTime;
+		m_cpuTime -= m_subtractedGpuTime;
 	}
 
 	void FramePassTimer::reset()
@@ -116,90 +110,67 @@ namespace crg
 		m_gpuTime = 0ns;
 	}
 
-	void FramePassTimer::beginPass( VkCommandBuffer commandBuffer
-		, uint32_t passIndex )const
+	void FramePassTimer::beginPass( VkCommandBuffer commandBuffer )
 	{
-		assert( passIndex < m_passesCount );
+		std::swap( m_queries.front(), m_queries.back() );
+		auto & query = m_queries.front();
 		m_context.vkCmdResetQueryPool( commandBuffer
 			, m_timerQueries
-			, m_baseQueryOffset + passIndex * 2u
+			, query.offset
 			, 2u );
 		m_context.vkCmdWriteTimestamp( commandBuffer
 			, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT
 			, m_timerQueries
-			, m_baseQueryOffset + passIndex * 2u + 0u );
+			, query.offset + 0u );
 	}
 
-	void FramePassTimer::endPass( VkCommandBuffer commandBuffer
-		, uint32_t passIndex )const
+	void FramePassTimer::endPass( VkCommandBuffer commandBuffer )
 	{
-		assert( passIndex < m_passesCount );
+		auto & query = m_queries.front();
 		m_context.vkCmdWriteTimestamp( commandBuffer
 			, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT
 			, m_timerQueries
-			, m_baseQueryOffset + passIndex * 2u + 1u );
+			, query.offset + 1u );
+		query.written = true;
 	}
 
 	void FramePassTimer::retrieveGpuTime()
 	{
 		static float const period = m_context.properties.limits.timestampPeriod;
+
 		auto before = Clock::now();
 		m_gpuTime = 0ns;
-		m_subtracteGpuTime = 0ns;
+		m_subtractedGpuTime = 0ns;
+		auto & query = m_queries.front();
 
-		for ( uint32_t i = 0; i < m_passesCount; ++i )
+		if ( query.started && query.written )
 		{
-			auto & started = m_startedPasses[i];
+			std::vector< uint64_t > values{ 0u, 0u };
+			m_context.vkGetQueryPoolResults( m_context.device
+				, m_timerQueries
+				, query.offset
+				, 2u
+				, sizeof( uint64_t ) * values.size()
+				, values.data()
+				, sizeof( uint64_t )
+				, VK_QUERY_RESULT_WAIT_BIT | VK_QUERY_RESULT_64_BIT );
+
+			auto gpuTime = Nanoseconds{ uint64_t( float( values[1] - values[0] ) / period ) };
+			m_gpuTime += gpuTime;
 
-			if ( started.first )
+			if ( query.subtractGpuFromCpu )
 			{
-				std::vector< uint64_t > values{ 0u, 0u };
-				m_context.vkGetQueryPoolResults( m_context.device
-					, m_timerQueries
-					, m_baseQueryOffset + i * 2u
-					, 2u
-					, sizeof( uint64_t ) * values.size()
-					, values.data()
-					, sizeof( uint64_t )
-					, VK_QUERY_RESULT_WAIT_BIT | VK_QUERY_RESULT_64_BIT );
-
-				auto gpuTime = Nanoseconds{ uint64_t( float( values[1] - values[0] ) / period ) };
-				m_gpuTime += gpuTime;
-
-				if ( started.second )
-				{
-					m_subtracteGpuTime += gpuTime;
-				}
-
-				started.first = false;
-				started.second = false;
+				m_subtractedGpuTime += gpuTime;
 			}
+
+			query.started = false;
+			query.subtractGpuFromCpu = false;
+			query.written = false;
 		}
 
 		auto after = Clock::now();
 		m_cpuTime += ( after - before );
 	}
 
-	void FramePassTimer::updateCount( uint32_t count )
-	{
-		if ( m_passesCount != count )
-		{
-			m_passesCount = count;
-
-			if ( m_ownPool )
-			{
-				crgUnregisterObject( m_context, m_timerQueries );
-				m_context.vkDestroyQueryPool( m_context.device
-					, m_timerQueries
-					, m_context.allocator );
-				m_timerQueries = createQueryPool( m_context
-					, m_name
-					, m_passesCount );
-			}
-
-			m_startedPasses.resize( m_passesCount );
-		}
-	}
-
 	//*********************************************************************************************
 }

source/RenderGraph/RunnableGraph.cpp

@@ -164,7 +164,7 @@ namespace crg
 		, m_nodes{ std::move( nodes ) }
 		, m_rootNode{ std::move( rootNode ) }
 		, m_timerQueries{ m_context
-			, createQueryPool( m_context, m_graph.getName() + "TimerQueries", uint32_t( ( m_nodes.size() + 1u ) * 2u ) )
+			, createQueryPool( m_context, m_graph.getName() + "TimerQueries", uint32_t( ( m_nodes.size() + 1u ) * 2u * 2u ) )
 			, []( GraphContext & ctx, VkQueryPool & object )
 			{
 				crgUnregisterObject( ctx, object );

source/RenderGraph/RunnablePass.cpp

@@ -633,6 +633,11 @@ namespace crg
 		}
 	}
 
+	void RunnablePass::notifyPassRender()
+	{
+		m_timer.notifyPassRender();
+	}
+
 	VkCommandBuffer RunnablePass::doCreateCommandBuffer( std::string const & suffix )
 	{
 		VkCommandBuffer result{};