diff --git a/layer_gpu_profile/source/layer_device_functions.hpp b/layer_gpu_profile/source/layer_device_functions.hpp index d0eb6a3..1c71367 100644 --- a/layer_gpu_profile/source/layer_device_functions.hpp +++ b/layer_gpu_profile/source/layer_device_functions.hpp @@ -1,7 +1,7 @@ /* * SPDX-License-Identifier: MIT * ---------------------------------------------------------------------------- - * Copyright (c) 2024-2025 Arm Limited + * Copyright (c) 2024-2026 Arm Limited * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -261,6 +261,14 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdFillBuffer(VkCommandBuffer comma VkDeviceSize size, uint32_t data); +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdFillMemoryKHR( + VkCommandBuffer commandBuffer, + const VkDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + uint32_t data); + /* See Vulkan API for documentation. */ template<> VKAPI_ATTR void VKAPI_CALL layer_vkCmdClearColorImage(VkCommandBuffer commandBuffer, @@ -359,6 +367,56 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyImageToBuffer2KHR(VkCommandBuffer commandBuffer, const VkCopyImageToBufferInfo2* pCopyImageToBufferInfo); +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyImageToMemoryKHR( + VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryImageInfoKHR* pCopyMemoryInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyMemoryKHR( + VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryInfoKHR* pCopyMemoryInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyMemoryToImageKHR( + VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryImageInfoKHR* pCopyMemoryInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyQueryPoolResultsToMemoryKHR( + VkCommandBuffer commandBuffer, + VkQueryPool queryPool, + uint32_t firstQuery, + uint32_t queryCount, + const VkStridedDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + VkQueryResultFlags queryResultFlags); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdUpdateMemoryKHR( + VkCommandBuffer commandBuffer, + const VkDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + VkDeviceSize dataSize, + const void* pData); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyMemoryIndirectKHR(VkCommandBuffer commandBuffer, + const VkCopyMemoryIndirectInfoKHR* pCopyMemoryIndirectInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyMemoryToImageIndirectKHR(VkCommandBuffer commandBuffer, + const VkCopyMemoryToImageIndirectInfoKHR* pCopyMemoryToImageIndirectInfo); + /* See Vulkan API for documentation. */ template<> VKAPI_ATTR void VKAPI_CALL diff --git a/layer_gpu_profile/source/layer_device_functions_transfer.cpp b/layer_gpu_profile/source/layer_device_functions_transfer.cpp index 540a883..5d4b3c7 100644 --- a/layer_gpu_profile/source/layer_device_functions_transfer.cpp +++ b/layer_gpu_profile/source/layer_device_functions_transfer.cpp @@ -1,7 +1,7 @@ /* * SPDX-License-Identifier: MIT * ---------------------------------------------------------------------------- - * Copyright (c) 2024-2025 Arm Limited + * Copyright (c) 2024-2026 Arm Limited * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -127,7 +127,10 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdFillBuffer(VkCommandBuffer comma byteCount = -2; } - registerBufferTransfer(layer, commandBuffer, Tracker::LCSBufferTransfer::Type::fill_buffer, byteCount); + registerBufferTransfer(layer, + commandBuffer, + Tracker::LCSBufferTransfer::Type::fill_buffer, + byteCount); // Release the lock to call into the driver lock.unlock(); @@ -135,6 +138,32 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdFillBuffer(VkCommandBuffer comma emitCPUTrap(*layer, commandBuffer); } +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdFillMemoryKHR( + VkCommandBuffer commandBuffer, + const VkDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + uint32_t data +) { + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + int64_t byteCount = static_cast(pDstRange->size); + registerBufferTransfer(layer, + commandBuffer, + Tracker::LCSBufferTransfer::Type::fill_buffer, + byteCount); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdFillMemoryKHR(commandBuffer, pDstRange, dstFlags, data); + emitCPUTrap(*layer, commandBuffer); +} + /* See Vulkan API for documentation. */ template<> VKAPI_ATTR void VKAPI_CALL layer_vkCmdClearColorImage(VkCommandBuffer commandBuffer, @@ -560,11 +589,210 @@ VKAPI_ATTR void VKAPI_CALL emitCPUTrap(*layer, commandBuffer); } +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyImageToMemoryKHR( + VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryImageInfoKHR* pCopyMemoryInfo +) { + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + int64_t pixelCount = 0; + for (uint32_t i = 0; i < pCopyMemoryInfo->regionCount; i++) + { + const auto& extent = pCopyMemoryInfo->pRegions[i].imageExtent; + int64_t regionPixelCount = static_cast(extent.width) + * static_cast(extent.height) + * static_cast(extent.depth); + pixelCount += regionPixelCount; + } + + registerImageTransfer(layer, + commandBuffer, + Tracker::LCSImageTransfer::Type::copy_image_to_buffer, + pixelCount); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdCopyImageToMemoryKHR(commandBuffer, pCopyMemoryInfo); + emitCPUTrap(*layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyMemoryKHR( + VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryInfoKHR* pCopyMemoryInfo +) { + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + int64_t byteCount = 0; + for (uint32_t i = 0; i < pCopyMemoryInfo->regionCount; i++) + { + byteCount += static_cast(pCopyMemoryInfo->pRegions[i].dstRange.size); + } + + registerBufferTransfer(layer, + commandBuffer, + Tracker::LCSBufferTransfer::Type::copy_memory, + byteCount); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdCopyMemoryKHR(commandBuffer, pCopyMemoryInfo); + emitCPUTrap(*layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyMemoryToImageKHR( + VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryImageInfoKHR* pCopyMemoryInfo +) { + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + int64_t pixelCount = 0; + for (uint32_t i = 0; i < pCopyMemoryInfo->regionCount; i++) + { + const auto& extent = pCopyMemoryInfo->pRegions[i].imageExtent; + int64_t regionPixelCount = static_cast(extent.width) + * static_cast(extent.height) + * static_cast(extent.depth); + pixelCount += regionPixelCount; + } + + registerImageTransfer(layer, + commandBuffer, + Tracker::LCSImageTransfer::Type::copy_memory_to_image, + pixelCount); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdCopyMemoryToImageKHR(commandBuffer, pCopyMemoryInfo); + emitCPUTrap(*layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyQueryPoolResultsToMemoryKHR( + VkCommandBuffer commandBuffer, + VkQueryPool queryPool, + uint32_t firstQuery, + uint32_t queryCount, + const VkStridedDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + VkQueryResultFlags queryResultFlags +) { + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + int64_t byteCount = static_cast(pDstRange->size); + registerBufferTransfer(layer, + commandBuffer, + Tracker::LCSBufferTransfer::Type::unknown, + byteCount); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdCopyQueryPoolResultsToMemoryKHR( + commandBuffer, queryPool, firstQuery, queryCount, pDstRange, dstFlags, queryResultFlags); + emitCPUTrap(*layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdUpdateMemoryKHR( + VkCommandBuffer commandBuffer, + const VkDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + VkDeviceSize dataSize, + const void* pData +) { + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + int64_t byteCount = static_cast(dataSize); + registerBufferTransfer(layer, + commandBuffer, + Tracker::LCSBufferTransfer::Type::copy_memory, + byteCount); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdUpdateMemoryKHR(commandBuffer, pDstRange, dstFlags, dataSize, pData); + emitCPUTrap(*layer, commandBuffer); +} + /* See Vulkan API for documentation. */ template<> VKAPI_ATTR void VKAPI_CALL - layer_vkCmdCopyAccelerationStructureKHR(VkCommandBuffer commandBuffer, - const VkCopyAccelerationStructureInfoKHR* pInfo) + layer_vkCmdCopyMemoryIndirectKHR(VkCommandBuffer commandBuffer, + const VkCopyMemoryIndirectInfoKHR* pCopyMemoryIndirectInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + // The transfer size is stored in device memory, so it is not known here. + registerBufferTransfer(layer, + commandBuffer, + Tracker::LCSBufferTransfer::Type::copy_memory, + -1); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdCopyMemoryIndirectKHR(commandBuffer, pCopyMemoryIndirectInfo); + emitCPUTrap(*layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyMemoryToImageIndirectKHR(VkCommandBuffer commandBuffer, + const VkCopyMemoryToImageIndirectInfoKHR* pCopyMemoryToImageIndirectInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + // The transfer size is stored in device memory, so it is not known here. + registerImageTransfer(layer, + commandBuffer, + Tracker::LCSImageTransfer::Type::copy_memory_to_image, + -1); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdCopyMemoryToImageIndirectKHR(commandBuffer, pCopyMemoryToImageIndirectInfo); + emitCPUTrap(*layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyAccelerationStructureKHR(VkCommandBuffer commandBuffer, + const VkCopyAccelerationStructureInfoKHR* pInfo) { LAYER_TRACE(__func__); diff --git a/layer_gpu_support/README_LAYER.md b/layer_gpu_support/README_LAYER.md index ab740a4..eef3b6d 100644 --- a/layer_gpu_support/README_LAYER.md +++ b/layer_gpu_support/README_LAYER.md @@ -121,18 +121,22 @@ convenience options to force disable or enable all serialization. "queue": false, // Force cross-queue serialization of submits "queue_wait_idle": false, // Insert vkDeviceWaitIdle after submits "commandstream": { + "asbuild": { + "pre": false, // Insert full barrier before acceleration structure builds + "post": false // Insert full barrier after acceleration structure builds + }, "compute": { "pre": false, // Insert full barrier before dispatches "post": false // Insert full barrier after dispatches }, + "datagraph": { + "pre": false, // Insert full barrier before data graph dispatches + "post": false // Insert full barrier after data graph dispatches + }, "renderpass": { "pre": false, // Insert full barrier before render passes "post": false // Insert full barrier after render passes }, - "asbuild": { - "pre": false, // Insert full barrier before acceleration structure builds - "post": false // Insert full barrier after acceleration structure builds - }, "tracerays": { "pre": false, // Insert full barrier before trace rays "post": false // Insert full barrier after trace rays @@ -190,4 +194,4 @@ allocated and handled by the driver. - - - -_Copyright © 2025, Arm Limited and contributors._ +_Copyright © 2025-2026, Arm Limited and contributors._ diff --git a/layer_gpu_support/layer_config.json b/layer_gpu_support/layer_config.json index 726e999..b56686e 100644 --- a/layer_gpu_support/layer_config.json +++ b/layer_gpu_support/layer_config.json @@ -8,15 +8,19 @@ "all": false, "none": false, "commandstream": { + "asbuild": { + "pre": false, + "post": false + }, "compute": { "pre": false, "post": false }, - "renderpass": { + "datagraph": { "pre": false, "post": false }, - "asbuild": { + "renderpass": { "pre": false, "post": false }, diff --git a/layer_gpu_support/source/CMakeLists.txt b/layer_gpu_support/source/CMakeLists.txt index 4b224de..34fe334 100644 --- a/layer_gpu_support/source/CMakeLists.txt +++ b/layer_gpu_support/source/CMakeLists.txt @@ -1,6 +1,6 @@ # SPDX-License-Identifier: MIT # ----------------------------------------------------------------------------- -# Copyright (c) 2025 Arm Limited +# Copyright (c) 2025-2026 Arm Limited # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to @@ -46,6 +46,7 @@ add_library( instance.cpp layer_config.cpp layer_device_functions_dispatch.cpp + layer_device_functions_data_graph.cpp layer_device_functions_image.cpp layer_device_functions_pipelines.cpp layer_device_functions_queue.cpp diff --git a/layer_gpu_support/source/layer_config.cpp b/layer_gpu_support/source/layer_config.cpp index a66c1c3..52dfc50 100644 --- a/layer_gpu_support/source/layer_config.cpp +++ b/layer_gpu_support/source/layer_config.cpp @@ -1,7 +1,7 @@ /* * SPDX-License-Identifier: MIT * ---------------------------------------------------------------------------- - * Copyright (c) 2025 Arm Limited + * Copyright (c) 2025-2026 Arm Limited * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -86,6 +86,9 @@ void LayerConfig::parse_serialization_options(const json& config) bool s_stream_c_pre = s_stream.at("compute").at("pre"); bool s_stream_c_post = s_stream.at("compute").at("post"); + bool s_stream_dg_pre = s_stream.at("datagraph").at("pre"); + bool s_stream_dg_post = s_stream.at("datagraph").at("post"); + bool s_stream_rp_pre = s_stream.at("renderpass").at("pre"); bool s_stream_rp_post = s_stream.at("renderpass").at("post"); @@ -108,6 +111,9 @@ void LayerConfig::parse_serialization_options(const json& config) conf_serialize_dispatch_pre = (!s_none) && (s_all || s_stream_c_pre); conf_serialize_dispatch_post = (!s_none) && (s_all || s_stream_c_post); + conf_serialize_data_graph_dispatch_pre = (!s_none) && (s_all || s_stream_dg_pre); + conf_serialize_data_graph_dispatch_post = (!s_none) && (s_all || s_stream_dg_post); + conf_serialize_render_pass_pre = (!s_none) && (s_all || s_stream_rp_pre); conf_serialize_render_pass_post = (!s_none) && (s_all || s_stream_rp_post); @@ -126,6 +132,8 @@ void LayerConfig::parse_serialization_options(const json& config) LAYER_LOG(" - Wait idle after queue submit: %d", conf_serialize_queue_wait_idle); LAYER_LOG(" - Serialize compute pre: %d", conf_serialize_dispatch_pre); LAYER_LOG(" - Serialize compute post: %d", conf_serialize_dispatch_post); + LAYER_LOG(" - Serialize data graph pre: %d", conf_serialize_data_graph_dispatch_pre); + LAYER_LOG(" - Serialize data graph post: %d", conf_serialize_data_graph_dispatch_post); LAYER_LOG(" - Serialize render pass pre: %d", conf_serialize_render_pass_pre); LAYER_LOG(" - Serialize render pass post: %d", conf_serialize_render_pass_post); LAYER_LOG(" - Serialize acceleration structure build pre: %d", conf_serialize_as_build_pre); @@ -347,6 +355,18 @@ bool LayerConfig::serialize_cmdstream_compute_dispatch_post() const return conf_serialize_dispatch_post; } +/* See header for documentation. */ +bool LayerConfig::serialize_cmdstream_data_graph_dispatch_pre() const +{ + return conf_serialize_data_graph_dispatch_pre; +} + +/* See header for documentation. */ +bool LayerConfig::serialize_cmdstream_data_graph_dispatch_post() const +{ + return conf_serialize_data_graph_dispatch_post; +} + /* See header for documentation. */ bool LayerConfig::serialize_cmdstream_render_pass_pre() const { diff --git a/layer_gpu_support/source/layer_config.hpp b/layer_gpu_support/source/layer_config.hpp index c1a0cee..70d4e72 100644 --- a/layer_gpu_support/source/layer_config.hpp +++ b/layer_gpu_support/source/layer_config.hpp @@ -1,7 +1,7 @@ /* * SPDX-License-Identifier: MIT * ---------------------------------------------------------------------------- - * Copyright (c) 2025 Arm Limited + * Copyright (c) 2025-2026 Arm Limited * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -83,6 +83,16 @@ class LayerConfig */ bool serialize_cmdstream_compute_dispatch_post() const; + /** + * @brief True if config wants to serialize before data graph workloads. + */ + bool serialize_cmdstream_data_graph_dispatch_pre() const; + + /** + * @brief True if config wants to serialize after data graph workloads. + */ + bool serialize_cmdstream_data_graph_dispatch_post() const; + /** * @brief True if config wants to serialize before render pass workloads. */ @@ -233,6 +243,16 @@ class LayerConfig */ bool conf_serialize_dispatch_post {false}; + /** + * @brief True if we force serialize before data graph dispatches. + */ + bool conf_serialize_data_graph_dispatch_pre {false}; + + /** + * @brief True if we force serialize after data graph dispatches. + */ + bool conf_serialize_data_graph_dispatch_post {false}; + /** * @brief True if we force serialize before render pass workloads. */ diff --git a/layer_gpu_support/source/layer_device_functions.hpp b/layer_gpu_support/source/layer_device_functions.hpp index 0343ff7..d068743 100644 --- a/layer_gpu_support/source/layer_device_functions.hpp +++ b/layer_gpu_support/source/layer_device_functions.hpp @@ -1,7 +1,7 @@ /* * SPDX-License-Identifier: MIT * ---------------------------------------------------------------------------- - * Copyright (c) 2025 Arm Limited + * Copyright (c) 2025-2026 Arm Limited * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -104,6 +104,14 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdDispatchIndirect(VkCommandBuffer VkBuffer buffer, VkDeviceSize offset); +// Commands for data graph + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdDispatchDataGraphARM(VkCommandBuffer commandBuffer, + VkDataGraphPipelineSessionARM session, + const VkDataGraphPipelineDispatchInfoARM* pInfo); + // Commands for trace rays /* See Vulkan API for documentation. */ @@ -173,6 +181,13 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdFillBuffer(VkCommandBuffer comma VkDeviceSize size, uint32_t data); +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdFillMemoryKHR(VkCommandBuffer commandBuffer, + const VkDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + uint32_t data); + /* See Vulkan API for documentation. */ template<> VKAPI_ATTR void VKAPI_CALL layer_vkCmdClearColorImage(VkCommandBuffer commandBuffer, @@ -271,6 +286,54 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyImageToBuffer2KHR(VkCommandBuffer commandBuffer, const VkCopyImageToBufferInfo2* pCopyImageToBufferInfo); +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyImageToMemoryKHR(VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryImageInfoKHR* pCopyMemoryInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyMemoryKHR(VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryInfoKHR* pCopyMemoryInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyMemoryToImageKHR(VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryImageInfoKHR* pCopyMemoryInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyQueryPoolResultsToMemoryKHR(VkCommandBuffer commandBuffer, + VkQueryPool queryPool, + uint32_t firstQuery, + uint32_t queryCount, + const VkStridedDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + VkQueryResultFlags queryResultFlags); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdUpdateMemoryKHR(VkCommandBuffer commandBuffer, + const VkDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + VkDeviceSize dataSize, + const void* pData); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyMemoryIndirectKHR(VkCommandBuffer commandBuffer, + const VkCopyMemoryIndirectInfoKHR* pCopyMemoryIndirectInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyMemoryToImageIndirectKHR(VkCommandBuffer commandBuffer, + const VkCopyMemoryToImageIndirectInfoKHR* pCopyMemoryToImageIndirectInfo); + /* See Vulkan API for documentation. */ template<> VKAPI_ATTR void VKAPI_CALL diff --git a/layer_gpu_support/source/layer_device_functions_data_graph.cpp b/layer_gpu_support/source/layer_device_functions_data_graph.cpp new file mode 100644 index 0000000..e5af061 --- /dev/null +++ b/layer_gpu_support/source/layer_device_functions_data_graph.cpp @@ -0,0 +1,103 @@ +/* + * SPDX-License-Identifier: MIT + * ---------------------------------------------------------------------------- + * Copyright (c) 2026 Arm Limited + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * ---------------------------------------------------------------------------- + */ + +#include "device.hpp" +#include "framework/device_dispatch_table.hpp" + +#include + +extern std::mutex g_vulkanLock; + +/** + * @brief Pre-data-graph-dispatch code injection point. + * + * @param layer The layer context for the device. + * @param commandBuffer The command buffer we are recording. + */ +static void preDispatchDataGraph(Device* layer, VkCommandBuffer commandBuffer) +{ + if (!layer->instance->config.serialize_cmdstream_data_graph_dispatch_pre()) + { + return; + } + + // Execution dependency + layer->driver.vkCmdPipelineBarrier(commandBuffer, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + 0, + 0, + nullptr, + 0, + nullptr, + 0, + nullptr); +} + +/** + * @brief Post-data-graph-dispatch code injection point. + * + * @param layer The layer context for the device. + * @param commandBuffer The command buffer we are recording. + */ +static void postDispatchDataGraph(Device* layer, VkCommandBuffer commandBuffer) +{ + if (!layer->instance->config.serialize_cmdstream_data_graph_dispatch_post()) + { + return; + } + + // Execution dependency + layer->driver.vkCmdPipelineBarrier(commandBuffer, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + 0, + 0, + nullptr, + 0, + nullptr, + 0, + nullptr); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdDispatchDataGraphARM(VkCommandBuffer commandBuffer, + VkDataGraphPipelineSessionARM session, + const VkDataGraphPipelineDispatchInfoARM* pInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + + preDispatchDataGraph(layer, commandBuffer); + layer->driver.vkCmdDispatchDataGraphARM(commandBuffer, session, pInfo); + postDispatchDataGraph(layer, commandBuffer); +} diff --git a/layer_gpu_support/source/layer_device_functions_transfer.cpp b/layer_gpu_support/source/layer_device_functions_transfer.cpp index bea1605..95e4555 100644 --- a/layer_gpu_support/source/layer_device_functions_transfer.cpp +++ b/layer_gpu_support/source/layer_device_functions_transfer.cpp @@ -1,7 +1,7 @@ /* * SPDX-License-Identifier: MIT * ---------------------------------------------------------------------------- - * Copyright (c) 2025 Arm Limited + * Copyright (c) 2025-2026 Arm Limited * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -106,6 +106,27 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdFillBuffer(VkCommandBuffer comma postTransfer(layer, commandBuffer); } +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdFillMemoryKHR(VkCommandBuffer commandBuffer, + const VkDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + uint32_t data) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + + preTransfer(layer, commandBuffer); + layer->driver.vkCmdFillMemoryKHR(commandBuffer, pDstRange, dstFlags, data); + postTransfer(layer, commandBuffer); +} + /* See Vulkan API for documentation. */ template<> VKAPI_ATTR void VKAPI_CALL layer_vkCmdClearColorImage(VkCommandBuffer commandBuffer, @@ -401,6 +422,153 @@ VKAPI_ATTR void VKAPI_CALL postTransfer(layer, commandBuffer); } +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyImageToMemoryKHR(VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryImageInfoKHR* pCopyMemoryInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + + preTransfer(layer, commandBuffer); + layer->driver.vkCmdCopyImageToMemoryKHR(commandBuffer, pCopyMemoryInfo); + postTransfer(layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyMemoryKHR(VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryInfoKHR* pCopyMemoryInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + + preTransfer(layer, commandBuffer); + layer->driver.vkCmdCopyMemoryKHR(commandBuffer, pCopyMemoryInfo); + postTransfer(layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyMemoryToImageKHR(VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryImageInfoKHR* pCopyMemoryInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + + preTransfer(layer, commandBuffer); + layer->driver.vkCmdCopyMemoryToImageKHR(commandBuffer, pCopyMemoryInfo); + postTransfer(layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyQueryPoolResultsToMemoryKHR(VkCommandBuffer commandBuffer, + VkQueryPool queryPool, + uint32_t firstQuery, + uint32_t queryCount, + const VkStridedDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + VkQueryResultFlags queryResultFlags) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + + preTransfer(layer, commandBuffer); + layer->driver.vkCmdCopyQueryPoolResultsToMemoryKHR( + commandBuffer, queryPool, firstQuery, queryCount, pDstRange, dstFlags, queryResultFlags); + postTransfer(layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdUpdateMemoryKHR(VkCommandBuffer commandBuffer, + const VkDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + VkDeviceSize dataSize, + const void* pData) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + + preTransfer(layer, commandBuffer); + layer->driver.vkCmdUpdateMemoryKHR(commandBuffer, pDstRange, dstFlags, dataSize, pData); + postTransfer(layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyMemoryIndirectKHR(VkCommandBuffer commandBuffer, + const VkCopyMemoryIndirectInfoKHR* pCopyMemoryIndirectInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + + preTransfer(layer, commandBuffer); + layer->driver.vkCmdCopyMemoryIndirectKHR(commandBuffer, pCopyMemoryIndirectInfo); + postTransfer(layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyMemoryToImageIndirectKHR(VkCommandBuffer commandBuffer, + const VkCopyMemoryToImageIndirectInfoKHR* pCopyMemoryToImageIndirectInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + + preTransfer(layer, commandBuffer); + layer->driver.vkCmdCopyMemoryToImageIndirectKHR(commandBuffer, pCopyMemoryToImageIndirectInfo); + postTransfer(layer, commandBuffer); +} + /* See Vulkan API for documentation. */ template<> VKAPI_ATTR void VKAPI_CALL diff --git a/layer_gpu_timeline/source/layer_device_functions.hpp b/layer_gpu_timeline/source/layer_device_functions.hpp index afa2dd4..b9ff173 100644 --- a/layer_gpu_timeline/source/layer_device_functions.hpp +++ b/layer_gpu_timeline/source/layer_device_functions.hpp @@ -244,6 +244,50 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdDrawIndirectCountKHR(VkCommandBu uint32_t maxDrawCount, uint32_t stride); +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdDrawIndexedIndirect2KHR(VkCommandBuffer commandBuffer, + const VkDrawIndirect2InfoKHR* pInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdDrawIndexedIndirectCount2KHR(VkCommandBuffer commandBuffer, + const VkDrawIndirectCount2InfoKHR* pInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdDrawIndirect2KHR(VkCommandBuffer commandBuffer, + const VkDrawIndirect2InfoKHR* pInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdDrawIndirectByteCount2EXT(VkCommandBuffer commandBuffer, + uint32_t instanceCount, + uint32_t firstInstance, + const VkBindTransformFeedbackBuffer2InfoEXT* pCounterInfo, + uint32_t counterOffset, + uint32_t vertexStride); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdDrawIndirectCount2KHR(VkCommandBuffer commandBuffer, + const VkDrawIndirectCount2InfoKHR* pInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdDrawMeshTasksIndirect2EXT(VkCommandBuffer commandBuffer, + const VkDrawIndirect2InfoKHR* pInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdDrawMeshTasksIndirectCount2EXT(VkCommandBuffer commandBuffer, + const VkDrawIndirectCount2InfoKHR* pInfo); + // Functions for compute dispatches /* See Vulkan API for documentation. */ @@ -444,6 +488,49 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyImageToBuffer2KHR(VkCommandBuffer commandBuffer, const VkCopyImageToBufferInfo2* pCopyImageToBufferInfo); +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyImageToMemoryKHR(VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryImageInfoKHR* pCopyMemoryInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyMemoryKHR(VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryInfoKHR* pCopyMemoryInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyMemoryToImageKHR(VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryImageInfoKHR* pCopyMemoryInfo); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyQueryPoolResultsToMemoryKHR(VkCommandBuffer commandBuffer, + VkQueryPool queryPool, + uint32_t firstQuery, + uint32_t queryCount, + const VkStridedDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + VkQueryResultFlags queryResultFlags); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdFillMemoryKHR(VkCommandBuffer commandBuffer, + const VkDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + uint32_t data); + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdUpdateMemoryKHR(VkCommandBuffer commandBuffer, + const VkDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + VkDeviceSize dataSize, + const void* pData); + /* See Vulkan API for documentation. */ template<> VKAPI_ATTR void VKAPI_CALL diff --git a/layer_gpu_timeline/source/layer_device_functions_draw_call.cpp b/layer_gpu_timeline/source/layer_device_functions_draw_call.cpp index b1ab208..2493a47 100644 --- a/layer_gpu_timeline/source/layer_device_functions_draw_call.cpp +++ b/layer_gpu_timeline/source/layer_device_functions_draw_call.cpp @@ -1,7 +1,7 @@ /* * SPDX-License-Identifier: MIT * ---------------------------------------------------------------------------- - * Copyright (c) 2024 Arm Limited + * Copyright (c) 2024-2026 Arm Limited * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -264,3 +264,139 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdDrawIndirectCountKHR(VkCommandBu layer->driver .vkCmdDrawIndirectCountKHR(commandBuffer, buffer, offset, countBuffer, countBufferOffset, maxDrawCount, stride); } + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdDrawIndexedIndirect2KHR(VkCommandBuffer commandBuffer, + const VkDrawIndirect2InfoKHR* pInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + registerDrawCall(layer, commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdDrawIndexedIndirect2KHR(commandBuffer, pInfo); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdDrawIndexedIndirectCount2KHR(VkCommandBuffer commandBuffer, + const VkDrawIndirectCount2InfoKHR* pInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + registerDrawCall(layer, commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdDrawIndexedIndirectCount2KHR(commandBuffer, pInfo); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdDrawIndirect2KHR(VkCommandBuffer commandBuffer, + const VkDrawIndirect2InfoKHR* pInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + registerDrawCall(layer, commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdDrawIndirect2KHR(commandBuffer, pInfo); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdDrawIndirectByteCount2EXT(VkCommandBuffer commandBuffer, + uint32_t instanceCount, + uint32_t firstInstance, + const VkBindTransformFeedbackBuffer2InfoEXT* pCounterInfo, + uint32_t counterOffset, + uint32_t vertexStride) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + registerDrawCall(layer, commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdDrawIndirectByteCount2EXT( + commandBuffer, instanceCount, firstInstance, pCounterInfo, counterOffset, vertexStride); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdDrawIndirectCount2KHR(VkCommandBuffer commandBuffer, + const VkDrawIndirectCount2InfoKHR* pInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + registerDrawCall(layer, commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdDrawIndirectCount2KHR(commandBuffer, pInfo); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdDrawMeshTasksIndirect2EXT(VkCommandBuffer commandBuffer, + const VkDrawIndirect2InfoKHR* pInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + registerDrawCall(layer, commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdDrawMeshTasksIndirect2EXT(commandBuffer, pInfo); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdDrawMeshTasksIndirectCount2EXT(VkCommandBuffer commandBuffer, + const VkDrawIndirectCount2InfoKHR* pInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + registerDrawCall(layer, commandBuffer); + + // Release the lock to call into the driver + lock.unlock(); + layer->driver.vkCmdDrawMeshTasksIndirectCount2EXT(commandBuffer, pInfo); +} diff --git a/layer_gpu_timeline/source/layer_device_functions_transfer.cpp b/layer_gpu_timeline/source/layer_device_functions_transfer.cpp index d8a3e2a..cac10db 100644 --- a/layer_gpu_timeline/source/layer_device_functions_transfer.cpp +++ b/layer_gpu_timeline/source/layer_device_functions_transfer.cpp @@ -128,6 +128,32 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdFillBuffer(VkCommandBuffer comma emitEndTag(layer, commandBuffer); } +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdFillMemoryKHR(VkCommandBuffer commandBuffer, + const VkDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + uint32_t data) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + int64_t byteCount = static_cast(pDstRange->size); + uint64_t tagID = registerBufferTransfer(layer, + commandBuffer, + Tracker::LCSBufferTransfer::Type::fill_buffer, + byteCount); + + // Release the lock to call into the driver + lock.unlock(); + emitStartTag(layer, commandBuffer, tagID); + layer->driver.vkCmdFillMemoryKHR(commandBuffer, pDstRange, dstFlags, data); + emitEndTag(layer, commandBuffer); +} + /* See Vulkan API for documentation. */ template<> VKAPI_ATTR void VKAPI_CALL layer_vkCmdClearColorImage(VkCommandBuffer commandBuffer, @@ -582,6 +608,161 @@ VKAPI_ATTR void VKAPI_CALL emitEndTag(layer, commandBuffer); } +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyImageToMemoryKHR(VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryImageInfoKHR* pCopyMemoryInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + int64_t pixelCount = 0; + for (uint32_t i = 0; i < pCopyMemoryInfo->regionCount; i++) + { + const auto& extent = pCopyMemoryInfo->pRegions[i].imageExtent; + int64_t regionPixelCount = static_cast(extent.width) + * static_cast(extent.height) + * static_cast(extent.depth); + pixelCount += regionPixelCount; + } + + uint64_t tagID = registerImageTransfer(layer, + commandBuffer, + Tracker::LCSImageTransfer::Type::copy_image_to_buffer, + pixelCount); + + // Release the lock to call into the driver + lock.unlock(); + emitStartTag(layer, commandBuffer, tagID); + layer->driver.vkCmdCopyImageToMemoryKHR(commandBuffer, pCopyMemoryInfo); + emitEndTag(layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyMemoryKHR(VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryInfoKHR* pCopyMemoryInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + int64_t byteCount = 0; + for (uint32_t i = 0; i < pCopyMemoryInfo->regionCount; i++) + { + byteCount += static_cast(pCopyMemoryInfo->pRegions[i].dstRange.size); + } + + uint64_t tagID = registerBufferTransfer(layer, + commandBuffer, + Tracker::LCSBufferTransfer::Type::copy_memory, + byteCount); + + // Release the lock to call into the driver + lock.unlock(); + emitStartTag(layer, commandBuffer, tagID); + layer->driver.vkCmdCopyMemoryKHR(commandBuffer, pCopyMemoryInfo); + emitEndTag(layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyMemoryToImageKHR(VkCommandBuffer commandBuffer, + const VkCopyDeviceMemoryImageInfoKHR* pCopyMemoryInfo) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + int64_t pixelCount = 0; + for (uint32_t i = 0; i < pCopyMemoryInfo->regionCount; i++) + { + const auto& extent = pCopyMemoryInfo->pRegions[i].imageExtent; + int64_t regionPixelCount = static_cast(extent.width) + * static_cast(extent.height) + * static_cast(extent.depth); + pixelCount += regionPixelCount; + } + + uint64_t tagID = registerImageTransfer(layer, + commandBuffer, + Tracker::LCSImageTransfer::Type::copy_memory_to_image, + pixelCount); + + // Release the lock to call into the driver + lock.unlock(); + emitStartTag(layer, commandBuffer, tagID); + layer->driver.vkCmdCopyMemoryToImageKHR(commandBuffer, pCopyMemoryInfo); + emitEndTag(layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL + layer_vkCmdCopyQueryPoolResultsToMemoryKHR(VkCommandBuffer commandBuffer, + VkQueryPool queryPool, + uint32_t firstQuery, + uint32_t queryCount, + const VkStridedDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + VkQueryResultFlags queryResultFlags) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + int64_t byteCount = static_cast(pDstRange->size); + uint64_t tagID = registerBufferTransfer(layer, + commandBuffer, + Tracker::LCSBufferTransfer::Type::unknown, + byteCount); + + // Release the lock to call into the driver + lock.unlock(); + emitStartTag(layer, commandBuffer, tagID); + layer->driver.vkCmdCopyQueryPoolResultsToMemoryKHR( + commandBuffer, queryPool, firstQuery, queryCount, pDstRange, dstFlags, queryResultFlags); + emitEndTag(layer, commandBuffer); +} + +/* See Vulkan API for documentation. */ +template<> +VKAPI_ATTR void VKAPI_CALL layer_vkCmdUpdateMemoryKHR(VkCommandBuffer commandBuffer, + const VkDeviceAddressRangeKHR* pDstRange, + VkAddressCommandFlagsKHR dstFlags, + VkDeviceSize dataSize, + const void* pData) +{ + LAYER_TRACE(__func__); + + // Hold the lock to access layer-wide global store + std::unique_lock lock {g_vulkanLock}; + auto* layer = Device::retrieve(commandBuffer); + + int64_t byteCount = static_cast(dataSize); + uint64_t tagID = registerBufferTransfer(layer, + commandBuffer, + Tracker::LCSBufferTransfer::Type::copy_memory, + byteCount); + + // Release the lock to call into the driver + lock.unlock(); + emitStartTag(layer, commandBuffer, tagID); + layer->driver.vkCmdUpdateMemoryKHR(commandBuffer, pDstRange, dstFlags, dataSize, pData); + emitEndTag(layer, commandBuffer); +} + /* See Vulkan API for documentation. */ template <> VKAPI_ATTR void VKAPI_CALL layer_vkCmdCopyMemoryIndirectKHR(