vk_memory_manager: Improve memory manager and its API
Fix a bug where the memory allocator could leave gaps between commits. To fix this the allocation algorithm was reworked, although it's still short in number of lines of code. Rework the allocation API to self-contained movable objects instead of naively using an unique_ptr to do the job for us. Remove the VK prefix.
This commit is contained in:
ReinUsesLisp
parent
f728a504aa
commit
e996f1ad09
@ -150,8 +150,8 @@ VkSemaphore VKBlitScreen::Draw(const Tegra::FramebufferConfig& framebuffer, bool
|
||||
SetUniformData(data, framebuffer);
|
||||
SetVertexData(data, framebuffer);
|
||||
|
||||
auto map = buffer_commit->Map();
|
||||
std::memcpy(map.Address(), &data, sizeof(data));
|
||||
const std::span<u8> map = buffer_commit.Map();
|
||||
std::memcpy(map.data(), &data, sizeof(data));
|
||||
|
||||
if (!use_accelerated) {
|
||||
const u64 image_offset = GetRawImageOffset(framebuffer, image_index);
|
||||
@ -165,8 +165,8 @@ VkSemaphore VKBlitScreen::Draw(const Tegra::FramebufferConfig& framebuffer, bool
|
||||
constexpr u32 block_height_log2 = 4;
|
||||
const u32 bytes_per_pixel = GetBytesPerPixel(framebuffer);
|
||||
Tegra::Texture::UnswizzleTexture(
|
||||
std::span(map.Address() + image_offset, size_bytes), std::span(host_ptr, size_bytes),
|
||||
bytes_per_pixel, framebuffer.width, framebuffer.height, 1, block_height_log2, 0);
|
||||
map.subspan(image_offset, size_bytes), std::span(host_ptr, size_bytes), bytes_per_pixel,
|
||||
framebuffer.width, framebuffer.height, 1, block_height_log2, 0);
|
||||
|
||||
const VkBufferImageCopy copy{
|
||||
.bufferOffset = image_offset,
|
||||
@ -224,8 +224,6 @@ VkSemaphore VKBlitScreen::Draw(const Tegra::FramebufferConfig& framebuffer, bool
|
||||
VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, write_barrier);
|
||||
});
|
||||
}
|
||||
map.Release();
|
||||
|
||||
scheduler.Record([renderpass = *renderpass, framebuffer = *framebuffers[image_index],
|
||||
descriptor_set = descriptor_sets[image_index], buffer = *buffer,
|
||||
size = swapchain.GetSize(), pipeline = *pipeline,
|
||||
@ -642,7 +640,7 @@ void VKBlitScreen::ReleaseRawImages() {
|
||||
raw_images.clear();
|
||||
raw_buffer_commits.clear();
|
||||
buffer.reset();
|
||||
buffer_commit.reset();
|
||||
buffer_commit = MemoryCommit{};
|
||||
}
|
||||
|
||||
void VKBlitScreen::CreateStagingBuffer(const Tegra::FramebufferConfig& framebuffer) {
|
||||
|
||||
@ -104,14 +104,14 @@ private:
|
||||
vk::Sampler sampler;
|
||||
|
||||
vk::Buffer buffer;
|
||||
VKMemoryCommit buffer_commit;
|
||||
MemoryCommit buffer_commit;
|
||||
|
||||
std::vector<u64> resource_ticks;
|
||||
|
||||
std::vector<vk::Semaphore> semaphores;
|
||||
std::vector<vk::Image> raw_images;
|
||||
std::vector<vk::ImageView> raw_image_views;
|
||||
std::vector<VKMemoryCommit> raw_buffer_commits;
|
||||
std::vector<MemoryCommit> raw_buffer_commits;
|
||||
u32 raw_width = 0;
|
||||
u32 raw_height = 0;
|
||||
};
|
||||
|
||||
@ -37,10 +37,10 @@ constexpr VkAccessFlags TRANSFORM_FEEDBACK_WRITE_ACCESS =
|
||||
} // Anonymous namespace
|
||||
|
||||
Buffer::Buffer(const Device& device_, VKMemoryManager& memory_manager, VKScheduler& scheduler_,
|
||||
VKStagingBufferPool& staging_pool_, VAddr cpu_addr_, std::size_t size_)
|
||||
StagingBufferPool& staging_pool_, VAddr cpu_addr_, std::size_t size_)
|
||||
: BufferBlock{cpu_addr_, size_}, device{device_}, scheduler{scheduler_}, staging_pool{
|
||||
staging_pool_} {
|
||||
const VkBufferCreateInfo ci{
|
||||
buffer = device.GetLogical().CreateBuffer(VkBufferCreateInfo{
|
||||
.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
|
||||
.pNext = nullptr,
|
||||
.flags = 0,
|
||||
@ -49,22 +49,20 @@ Buffer::Buffer(const Device& device_, VKMemoryManager& memory_manager, VKSchedul
|
||||
.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
|
||||
.queueFamilyIndexCount = 0,
|
||||
.pQueueFamilyIndices = nullptr,
|
||||
};
|
||||
|
||||
buffer.handle = device.GetLogical().CreateBuffer(ci);
|
||||
buffer.commit = memory_manager.Commit(buffer.handle, false);
|
||||
});
|
||||
commit = memory_manager.Commit(buffer, false);
|
||||
}
|
||||
|
||||
Buffer::~Buffer() = default;
|
||||
|
||||
void Buffer::Upload(std::size_t offset, std::size_t data_size, const u8* data) {
|
||||
const auto& staging = staging_pool.GetUnusedBuffer(data_size, true);
|
||||
std::memcpy(staging.commit->Map(data_size), data, data_size);
|
||||
const auto& staging = staging_pool.Request(data_size, true);
|
||||
std::memcpy(staging.mapped_span.data(), data, data_size);
|
||||
|
||||
scheduler.RequestOutsideRenderPassOperationContext();
|
||||
|
||||
const VkBuffer handle = Handle();
|
||||
scheduler.Record([staging = *staging.handle, handle, offset, data_size,
|
||||
scheduler.Record([staging = staging.buffer, handle, offset, data_size,
|
||||
&device = device](vk::CommandBuffer cmdbuf) {
|
||||
const VkBufferMemoryBarrier read_barrier{
|
||||
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
|
||||
@ -100,12 +98,12 @@ void Buffer::Upload(std::size_t offset, std::size_t data_size, const u8* data) {
|
||||
}
|
||||
|
||||
void Buffer::Download(std::size_t offset, std::size_t data_size, u8* data) {
|
||||
const auto& staging = staging_pool.GetUnusedBuffer(data_size, true);
|
||||
auto staging = staging_pool.Request(data_size, true);
|
||||
scheduler.RequestOutsideRenderPassOperationContext();
|
||||
|
||||
const VkBuffer handle = Handle();
|
||||
scheduler.Record(
|
||||
[staging = *staging.handle, handle, offset, data_size](vk::CommandBuffer cmdbuf) {
|
||||
[staging = staging.buffer, handle, offset, data_size](vk::CommandBuffer cmdbuf) {
|
||||
const VkBufferMemoryBarrier barrier{
|
||||
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
|
||||
.pNext = nullptr,
|
||||
@ -126,7 +124,7 @@ void Buffer::Download(std::size_t offset, std::size_t data_size, u8* data) {
|
||||
});
|
||||
scheduler.Finish();
|
||||
|
||||
std::memcpy(data, staging.commit->Map(data_size), data_size);
|
||||
std::memcpy(data, staging.mapped_span.data(), data_size);
|
||||
}
|
||||
|
||||
void Buffer::CopyFrom(const Buffer& src, std::size_t src_offset, std::size_t dst_offset,
|
||||
@ -166,7 +164,7 @@ VKBufferCache::VKBufferCache(VideoCore::RasterizerInterface& rasterizer_,
|
||||
Tegra::MemoryManager& gpu_memory_, Core::Memory::Memory& cpu_memory_,
|
||||
const Device& device_, VKMemoryManager& memory_manager_,
|
||||
VKScheduler& scheduler_, VKStreamBuffer& stream_buffer_,
|
||||
VKStagingBufferPool& staging_pool_)
|
||||
StagingBufferPool& staging_pool_)
|
||||
: VideoCommon::BufferCache<Buffer, VkBuffer, VKStreamBuffer>{rasterizer_, gpu_memory_,
|
||||
cpu_memory_, stream_buffer_},
|
||||
device{device_}, memory_manager{memory_manager_}, scheduler{scheduler_}, staging_pool{
|
||||
@ -181,12 +179,12 @@ std::shared_ptr<Buffer> VKBufferCache::CreateBlock(VAddr cpu_addr, std::size_t s
|
||||
|
||||
VKBufferCache::BufferInfo VKBufferCache::GetEmptyBuffer(std::size_t size) {
|
||||
size = std::max(size, std::size_t(4));
|
||||
const auto& empty = staging_pool.GetUnusedBuffer(size, false);
|
||||
const auto& empty = staging_pool.Request(size, false);
|
||||
scheduler.RequestOutsideRenderPassOperationContext();
|
||||
scheduler.Record([size, buffer = *empty.handle](vk::CommandBuffer cmdbuf) {
|
||||
scheduler.Record([size, buffer = empty.buffer](vk::CommandBuffer cmdbuf) {
|
||||
cmdbuf.FillBuffer(buffer, 0, size, 0);
|
||||
});
|
||||
return {*empty.handle, 0, 0};
|
||||
return {empty.buffer, 0, 0};
|
||||
}
|
||||
|
||||
} // namespace Vulkan
|
||||
|
||||
@ -10,19 +10,19 @@
|
||||
#include "video_core/buffer_cache/buffer_cache.h"
|
||||
#include "video_core/renderer_vulkan/vk_memory_manager.h"
|
||||
#include "video_core/renderer_vulkan/vk_staging_buffer_pool.h"
|
||||
#include "video_core/renderer_vulkan/vk_memory_manager.h"
|
||||
#include "video_core/renderer_vulkan/vk_stream_buffer.h"
|
||||
#include "video_core/vulkan_common/vulkan_wrapper.h"
|
||||
|
||||
namespace Vulkan {
|
||||
|
||||
class Device;
|
||||
class VKMemoryManager;
|
||||
class VKScheduler;
|
||||
|
||||
class Buffer final : public VideoCommon::BufferBlock {
|
||||
public:
|
||||
explicit Buffer(const Device& device, VKMemoryManager& memory_manager, VKScheduler& scheduler,
|
||||
VKStagingBufferPool& staging_pool, VAddr cpu_addr_, std::size_t size_);
|
||||
StagingBufferPool& staging_pool, VAddr cpu_addr_, std::size_t size_);
|
||||
~Buffer();
|
||||
|
||||
void Upload(std::size_t offset, std::size_t data_size, const u8* data);
|
||||
@ -33,7 +33,7 @@ public:
|
||||
std::size_t copy_size);
|
||||
|
||||
VkBuffer Handle() const {
|
||||
return *buffer.handle;
|
||||
return *buffer;
|
||||
}
|
||||
|
||||
u64 Address() const {
|
||||
@ -43,9 +43,10 @@ public:
|
||||
private:
|
||||
const Device& device;
|
||||
VKScheduler& scheduler;
|
||||
VKStagingBufferPool& staging_pool;
|
||||
StagingBufferPool& staging_pool;
|
||||
|
||||
VKBuffer buffer;
|
||||
vk::Buffer buffer;
|
||||
MemoryCommit commit;
|
||||
};
|
||||
|
||||
class VKBufferCache final : public VideoCommon::BufferCache<Buffer, VkBuffer, VKStreamBuffer> {
|
||||
@ -54,7 +55,7 @@ public:
|
||||
Tegra::MemoryManager& gpu_memory, Core::Memory::Memory& cpu_memory,
|
||||
const Device& device, VKMemoryManager& memory_manager,
|
||||
VKScheduler& scheduler, VKStreamBuffer& stream_buffer,
|
||||
VKStagingBufferPool& staging_pool);
|
||||
StagingBufferPool& staging_pool);
|
||||
~VKBufferCache();
|
||||
|
||||
BufferInfo GetEmptyBuffer(std::size_t size) override;
|
||||
@ -66,7 +67,7 @@ private:
|
||||
const Device& device;
|
||||
VKMemoryManager& memory_manager;
|
||||
VKScheduler& scheduler;
|
||||
VKStagingBufferPool& staging_pool;
|
||||
StagingBufferPool& staging_pool;
|
||||
};
|
||||
|
||||
} // namespace Vulkan
|
||||
|
||||
@ -164,7 +164,7 @@ VkDescriptorSet VKComputePass::CommitDescriptorSet(
|
||||
|
||||
QuadArrayPass::QuadArrayPass(const Device& device_, VKScheduler& scheduler_,
|
||||
VKDescriptorPool& descriptor_pool_,
|
||||
VKStagingBufferPool& staging_buffer_pool_,
|
||||
StagingBufferPool& staging_buffer_pool_,
|
||||
VKUpdateDescriptorQueue& update_descriptor_queue_)
|
||||
: VKComputePass(device_, descriptor_pool_, BuildQuadArrayPassDescriptorSetLayoutBinding(),
|
||||
BuildQuadArrayPassDescriptorUpdateTemplateEntry(),
|
||||
@ -177,18 +177,18 @@ QuadArrayPass::~QuadArrayPass() = default;
|
||||
std::pair<VkBuffer, VkDeviceSize> QuadArrayPass::Assemble(u32 num_vertices, u32 first) {
|
||||
const u32 num_triangle_vertices = (num_vertices / 4) * 6;
|
||||
const std::size_t staging_size = num_triangle_vertices * sizeof(u32);
|
||||
auto& buffer = staging_buffer_pool.GetUnusedBuffer(staging_size, false);
|
||||
const auto staging_ref = staging_buffer_pool.Request(staging_size, false);
|
||||
|
||||
update_descriptor_queue.Acquire();
|
||||
update_descriptor_queue.AddBuffer(*buffer.handle, 0, staging_size);
|
||||
update_descriptor_queue.AddBuffer(staging_ref.buffer, 0, staging_size);
|
||||
const VkDescriptorSet set = CommitDescriptorSet(update_descriptor_queue);
|
||||
|
||||
scheduler.RequestOutsideRenderPassOperationContext();
|
||||
|
||||
ASSERT(num_vertices % 4 == 0);
|
||||
const u32 num_quads = num_vertices / 4;
|
||||
scheduler.Record([layout = *layout, pipeline = *pipeline, buffer = *buffer.handle, num_quads,
|
||||
first, set](vk::CommandBuffer cmdbuf) {
|
||||
scheduler.Record([layout = *layout, pipeline = *pipeline, buffer = staging_ref.buffer,
|
||||
num_quads, first, set](vk::CommandBuffer cmdbuf) {
|
||||
constexpr u32 dispatch_size = 1024;
|
||||
cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
|
||||
cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, layout, 0, set, {});
|
||||
@ -208,11 +208,11 @@ std::pair<VkBuffer, VkDeviceSize> QuadArrayPass::Assemble(u32 num_vertices, u32
|
||||
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
|
||||
VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, 0, {}, {barrier}, {});
|
||||
});
|
||||
return {*buffer.handle, 0};
|
||||
return {staging_ref.buffer, 0};
|
||||
}
|
||||
|
||||
Uint8Pass::Uint8Pass(const Device& device, VKScheduler& scheduler_,
|
||||
VKDescriptorPool& descriptor_pool, VKStagingBufferPool& staging_buffer_pool_,
|
||||
VKDescriptorPool& descriptor_pool, StagingBufferPool& staging_buffer_pool_,
|
||||
VKUpdateDescriptorQueue& update_descriptor_queue_)
|
||||
: VKComputePass(device, descriptor_pool, BuildInputOutputDescriptorSetBindings(),
|
||||
BuildInputOutputDescriptorUpdateTemplate(), {}, VULKAN_UINT8_COMP_SPV),
|
||||
@ -224,15 +224,15 @@ Uint8Pass::~Uint8Pass() = default;
|
||||
std::pair<VkBuffer, u64> Uint8Pass::Assemble(u32 num_vertices, VkBuffer src_buffer,
|
||||
u64 src_offset) {
|
||||
const u32 staging_size = static_cast<u32>(num_vertices * sizeof(u16));
|
||||
auto& buffer = staging_buffer_pool.GetUnusedBuffer(staging_size, false);
|
||||
const auto staging_ref = staging_buffer_pool.Request(staging_size, false);
|
||||
|
||||
update_descriptor_queue.Acquire();
|
||||
update_descriptor_queue.AddBuffer(src_buffer, src_offset, num_vertices);
|
||||
update_descriptor_queue.AddBuffer(*buffer.handle, 0, staging_size);
|
||||
update_descriptor_queue.AddBuffer(staging_ref.buffer, 0, staging_size);
|
||||
const VkDescriptorSet set = CommitDescriptorSet(update_descriptor_queue);
|
||||
|
||||
scheduler.RequestOutsideRenderPassOperationContext();
|
||||
scheduler.Record([layout = *layout, pipeline = *pipeline, buffer = *buffer.handle, set,
|
||||
scheduler.Record([layout = *layout, pipeline = *pipeline, buffer = staging_ref.buffer, set,
|
||||
num_vertices](vk::CommandBuffer cmdbuf) {
|
||||
constexpr u32 dispatch_size = 1024;
|
||||
cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
|
||||
@ -252,12 +252,12 @@ std::pair<VkBuffer, u64> Uint8Pass::Assemble(u32 num_vertices, VkBuffer src_buff
|
||||
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
|
||||
VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, 0, {}, barrier, {});
|
||||
});
|
||||
return {*buffer.handle, 0};
|
||||
return {staging_ref.buffer, 0};
|
||||
}
|
||||
|
||||
QuadIndexedPass::QuadIndexedPass(const Device& device_, VKScheduler& scheduler_,
|
||||
VKDescriptorPool& descriptor_pool_,
|
||||
VKStagingBufferPool& staging_buffer_pool_,
|
||||
StagingBufferPool& staging_buffer_pool_,
|
||||
VKUpdateDescriptorQueue& update_descriptor_queue_)
|
||||
: VKComputePass(device_, descriptor_pool_, BuildInputOutputDescriptorSetBindings(),
|
||||
BuildInputOutputDescriptorUpdateTemplate(),
|
||||
@ -286,15 +286,15 @@ std::pair<VkBuffer, u64> QuadIndexedPass::Assemble(
|
||||
const u32 num_tri_vertices = (num_vertices / 4) * 6;
|
||||
|
||||
const std::size_t staging_size = num_tri_vertices * sizeof(u32);
|
||||
auto& buffer = staging_buffer_pool.GetUnusedBuffer(staging_size, false);
|
||||
const auto staging_ref = staging_buffer_pool.Request(staging_size, false);
|
||||
|
||||
update_descriptor_queue.Acquire();
|
||||
update_descriptor_queue.AddBuffer(src_buffer, src_offset, input_size);
|
||||
update_descriptor_queue.AddBuffer(*buffer.handle, 0, staging_size);
|
||||
update_descriptor_queue.AddBuffer(staging_ref.buffer, 0, staging_size);
|
||||
const VkDescriptorSet set = CommitDescriptorSet(update_descriptor_queue);
|
||||
|
||||
scheduler.RequestOutsideRenderPassOperationContext();
|
||||
scheduler.Record([layout = *layout, pipeline = *pipeline, buffer = *buffer.handle, set,
|
||||
scheduler.Record([layout = *layout, pipeline = *pipeline, buffer = staging_ref.buffer, set,
|
||||
num_tri_vertices, base_vertex, index_shift](vk::CommandBuffer cmdbuf) {
|
||||
static constexpr u32 dispatch_size = 1024;
|
||||
const std::array push_constants = {base_vertex, index_shift};
|
||||
@ -317,7 +317,7 @@ std::pair<VkBuffer, u64> QuadIndexedPass::Assemble(
|
||||
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
|
||||
VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, 0, {}, barrier, {});
|
||||
});
|
||||
return {*buffer.handle, 0};
|
||||
return {staging_ref.buffer, 0};
|
||||
}
|
||||
|
||||
} // namespace Vulkan
|
||||
|
||||
@ -16,8 +16,8 @@
|
||||
namespace Vulkan {
|
||||
|
||||
class Device;
|
||||
class StagingBufferPool;
|
||||
class VKScheduler;
|
||||
class VKStagingBufferPool;
|
||||
class VKUpdateDescriptorQueue;
|
||||
|
||||
class VKComputePass {
|
||||
@ -45,7 +45,7 @@ class QuadArrayPass final : public VKComputePass {
|
||||
public:
|
||||
explicit QuadArrayPass(const Device& device_, VKScheduler& scheduler_,
|
||||
VKDescriptorPool& descriptor_pool_,
|
||||
VKStagingBufferPool& staging_buffer_pool_,
|
||||
StagingBufferPool& staging_buffer_pool_,
|
||||
VKUpdateDescriptorQueue& update_descriptor_queue_);
|
||||
~QuadArrayPass();
|
||||
|
||||
@ -53,7 +53,7 @@ public:
|
||||
|
||||
private:
|
||||
VKScheduler& scheduler;
|
||||
VKStagingBufferPool& staging_buffer_pool;
|
||||
StagingBufferPool& staging_buffer_pool;
|
||||
VKUpdateDescriptorQueue& update_descriptor_queue;
|
||||
};
|
||||
|
||||
@ -61,7 +61,7 @@ class Uint8Pass final : public VKComputePass {
|
||||
public:
|
||||
explicit Uint8Pass(const Device& device_, VKScheduler& scheduler_,
|
||||
VKDescriptorPool& descriptor_pool_,
|
||||
VKStagingBufferPool& staging_buffer_pool_,
|
||||
StagingBufferPool& staging_buffer_pool_,
|
||||
VKUpdateDescriptorQueue& update_descriptor_queue_);
|
||||
~Uint8Pass();
|
||||
|
||||
@ -69,7 +69,7 @@ public:
|
||||
|
||||
private:
|
||||
VKScheduler& scheduler;
|
||||
VKStagingBufferPool& staging_buffer_pool;
|
||||
StagingBufferPool& staging_buffer_pool;
|
||||
VKUpdateDescriptorQueue& update_descriptor_queue;
|
||||
};
|
||||
|
||||
@ -77,7 +77,7 @@ class QuadIndexedPass final : public VKComputePass {
|
||||
public:
|
||||
explicit QuadIndexedPass(const Device& device_, VKScheduler& scheduler_,
|
||||
VKDescriptorPool& descriptor_pool_,
|
||||
VKStagingBufferPool& staging_buffer_pool_,
|
||||
StagingBufferPool& staging_buffer_pool_,
|
||||
VKUpdateDescriptorQueue& update_descriptor_queue_);
|
||||
~QuadIndexedPass();
|
||||
|
||||
@ -87,7 +87,7 @@ public:
|
||||
|
||||
private:
|
||||
VKScheduler& scheduler;
|
||||
VKStagingBufferPool& staging_buffer_pool;
|
||||
StagingBufferPool& staging_buffer_pool;
|
||||
VKUpdateDescriptorQueue& update_descriptor_queue;
|
||||
};
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
// Refer to the license.txt file included.
|
||||
|
||||
#include <algorithm>
|
||||
#include <bit>
|
||||
#include <optional>
|
||||
#include <tuple>
|
||||
#include <vector>
|
||||
@ -16,92 +17,93 @@
|
||||
#include "video_core/vulkan_common/vulkan_wrapper.h"
|
||||
|
||||
namespace Vulkan {
|
||||
|
||||
namespace {
|
||||
struct Range {
|
||||
u64 begin;
|
||||
u64 end;
|
||||
|
||||
u64 GetAllocationChunkSize(u64 required_size) {
|
||||
static constexpr u64 sizes[] = {16ULL << 20, 32ULL << 20, 64ULL << 20, 128ULL << 20};
|
||||
auto it = std::lower_bound(std::begin(sizes), std::end(sizes), required_size);
|
||||
return it != std::end(sizes) ? *it : Common::AlignUp(required_size, 256ULL << 20);
|
||||
[[nodiscard]] bool Contains(u64 iterator, u64 size) const noexcept {
|
||||
return iterator < end && begin < iterator + size;
|
||||
}
|
||||
};
|
||||
|
||||
[[nodiscard]] u64 GetAllocationChunkSize(u64 required_size) {
|
||||
static constexpr std::array sizes{
|
||||
0x1000ULL << 10, 0x1400ULL << 10, 0x1800ULL << 10, 0x1c00ULL << 10, 0x2000ULL << 10,
|
||||
0x3200ULL << 10, 0x4000ULL << 10, 0x6000ULL << 10, 0x8000ULL << 10, 0xA000ULL << 10,
|
||||
0x10000ULL << 10, 0x18000ULL << 10, 0x20000ULL << 10,
|
||||
};
|
||||
static_assert(std::is_sorted(sizes.begin(), sizes.end()));
|
||||
|
||||
const auto it = std::ranges::lower_bound(sizes, required_size);
|
||||
return it != sizes.end() ? *it : Common::AlignUp(required_size, 4ULL << 20);
|
||||
}
|
||||
|
||||
} // Anonymous namespace
|
||||
|
||||
class VKMemoryAllocation final {
|
||||
class MemoryAllocation {
|
||||
public:
|
||||
explicit VKMemoryAllocation(const Device& device_, vk::DeviceMemory memory_,
|
||||
explicit MemoryAllocation(const Device& device_, vk::DeviceMemory memory_,
|
||||
VkMemoryPropertyFlags properties_, u64 allocation_size_, u32 type_)
|
||||
: device{device_}, memory{std::move(memory_)}, properties{properties_},
|
||||
allocation_size{allocation_size_}, shifted_type{ShiftType(type_)} {}
|
||||
|
||||
VKMemoryCommit Commit(VkDeviceSize commit_size, VkDeviceSize alignment) {
|
||||
auto found = TryFindFreeSection(free_iterator, allocation_size,
|
||||
static_cast<u64>(commit_size), static_cast<u64>(alignment));
|
||||
if (!found) {
|
||||
found = TryFindFreeSection(0, free_iterator, static_cast<u64>(commit_size),
|
||||
static_cast<u64>(alignment));
|
||||
if (!found) {
|
||||
[[nodiscard]] std::optional<MemoryCommit> Commit(VkDeviceSize size, VkDeviceSize alignment) {
|
||||
const std::optional<u64> alloc = FindFreeRegion(size, alignment);
|
||||
if (!alloc) {
|
||||
// Signal out of memory, it'll try to do more allocations.
|
||||
return nullptr;
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
auto commit = std::make_unique<VKMemoryCommitImpl>(device, this, memory, *found,
|
||||
*found + commit_size);
|
||||
commits.push_back(commit.get());
|
||||
|
||||
// Last commit's address is highly probable to be free.
|
||||
free_iterator = *found + commit_size;
|
||||
|
||||
return commit;
|
||||
const Range range{
|
||||
.begin = *alloc,
|
||||
.end = *alloc + size,
|
||||
};
|
||||
commits.insert(std::ranges::upper_bound(commits, *alloc, {}, &Range::begin), range);
|
||||
return std::make_optional<MemoryCommit>(device, this, *memory, *alloc, *alloc + size);
|
||||
}
|
||||
|
||||
void Free(const VKMemoryCommitImpl* commit) {
|
||||
ASSERT(commit);
|
||||
|
||||
const auto it = std::find(std::begin(commits), std::end(commits), commit);
|
||||
if (it == commits.end()) {
|
||||
UNREACHABLE_MSG("Freeing unallocated commit!");
|
||||
return;
|
||||
}
|
||||
void Free(u64 begin) {
|
||||
const auto it = std::ranges::find(commits, begin, &Range::begin);
|
||||
ASSERT_MSG(it != commits.end(), "Invalid commit");
|
||||
commits.erase(it);
|
||||
}
|
||||
|
||||
[[nodiscard]] std::span<u8> Map() {
|
||||
if (!memory_mapped_span.empty()) {
|
||||
return memory_mapped_span;
|
||||
}
|
||||
u8* const raw_pointer = memory.Map(0, allocation_size);
|
||||
memory_mapped_span = std::span<u8>(raw_pointer, allocation_size);
|
||||
return memory_mapped_span;
|
||||
}
|
||||
|
||||
/// Returns whether this allocation is compatible with the arguments.
|
||||
bool IsCompatible(VkMemoryPropertyFlags wanted_properties, u32 type_mask) const {
|
||||
[[nodiscard]] bool IsCompatible(VkMemoryPropertyFlags wanted_properties, u32 type_mask) const {
|
||||
return (wanted_properties & properties) && (type_mask & shifted_type) != 0;
|
||||
}
|
||||
|
||||
private:
|
||||
static constexpr u32 ShiftType(u32 type) {
|
||||
[[nodiscard]] static constexpr u32 ShiftType(u32 type) {
|
||||
return 1U << type;
|
||||
}
|
||||
|
||||
/// A memory allocator, it may return a free region between "start" and "end" with the solicited
|
||||
/// requirements.
|
||||
std::optional<u64> TryFindFreeSection(u64 start, u64 end, u64 size, u64 alignment) const {
|
||||
u64 iterator = Common::AlignUp(start, alignment);
|
||||
while (iterator + size <= end) {
|
||||
const u64 try_left = iterator;
|
||||
const u64 try_right = try_left + size;
|
||||
|
||||
bool overlap = false;
|
||||
for (const auto& commit : commits) {
|
||||
const auto [commit_left, commit_right] = commit->interval;
|
||||
if (try_left < commit_right && commit_left < try_right) {
|
||||
// There's an overlap, continue the search where the overlapping commit ends.
|
||||
iterator = Common::AlignUp(commit_right, alignment);
|
||||
overlap = true;
|
||||
[[nodiscard]] std::optional<u64> FindFreeRegion(u64 size, u64 alignment) noexcept {
|
||||
ASSERT(std::has_single_bit(alignment));
|
||||
const u64 alignment_log2 = std::countr_zero(alignment);
|
||||
std::optional<u64> candidate;
|
||||
u64 iterator = 0;
|
||||
auto commit = commits.begin();
|
||||
while (iterator + size <= allocation_size) {
|
||||
candidate = candidate.value_or(iterator);
|
||||
if (commit == commits.end()) {
|
||||
break;
|
||||
}
|
||||
if (commit->Contains(*candidate, size)) {
|
||||
candidate = std::nullopt;
|
||||
}
|
||||
if (!overlap) {
|
||||
// A free address has been found.
|
||||
return try_left;
|
||||
iterator = Common::AlignUpLog2(commit->end, alignment_log2);
|
||||
++commit;
|
||||
}
|
||||
}
|
||||
|
||||
// No free regions where found, return an empty optional.
|
||||
return std::nullopt;
|
||||
return candidate;
|
||||
}
|
||||
|
||||
const Device& device; ///< Vulkan device.
|
||||
@ -109,21 +111,52 @@ private:
|
||||
const VkMemoryPropertyFlags properties; ///< Vulkan properties.
|
||||
const u64 allocation_size; ///< Size of this allocation.
|
||||
const u32 shifted_type; ///< Stored Vulkan type of this allocation, shifted.
|
||||
|
||||
/// Hints where the next free region is likely going to be.
|
||||
u64 free_iterator{};
|
||||
|
||||
/// Stores all commits done from this allocation.
|
||||
std::vector<const VKMemoryCommitImpl*> commits;
|
||||
std::vector<Range> commits; ///< All commit ranges done from this allocation.
|
||||
std::span<u8> memory_mapped_span; ///< Memory mapped span. Empty if not queried before.
|
||||
};
|
||||
|
||||
MemoryCommit::MemoryCommit(const Device& device_, MemoryAllocation* allocation_,
|
||||
VkDeviceMemory memory_, u64 begin, u64 end) noexcept
|
||||
: device{&device_}, allocation{allocation_}, memory{memory_}, interval{begin, end} {}
|
||||
|
||||
MemoryCommit::~MemoryCommit() {
|
||||
Release();
|
||||
}
|
||||
|
||||
MemoryCommit& MemoryCommit::operator=(MemoryCommit&& rhs) noexcept {
|
||||
Release();
|
||||
device = rhs.device;
|
||||
allocation = std::exchange(rhs.allocation, nullptr);
|
||||
memory = rhs.memory;
|
||||
interval = rhs.interval;
|
||||
span = std::exchange(rhs.span, std::span<u8>{});
|
||||
return *this;
|
||||
}
|
||||
|
||||
MemoryCommit::MemoryCommit(MemoryCommit&& rhs) noexcept
|
||||
: device{rhs.device}, allocation{std::exchange(rhs.allocation, nullptr)}, memory{rhs.memory},
|
||||
interval{rhs.interval}, span{std::exchange(rhs.span, std::span<u8>{})} {}
|
||||
|
||||
std::span<u8> MemoryCommit::Map() {
|
||||
if (!span.empty()) {
|
||||
return span;
|
||||
}
|
||||
span = allocation->Map().subspan(interval.first, interval.second - interval.first);
|
||||
return span;
|
||||
}
|
||||
|
||||
void MemoryCommit::Release() {
|
||||
if (allocation) {
|
||||
allocation->Free(interval.first);
|
||||
}
|
||||
}
|
||||
|
||||
VKMemoryManager::VKMemoryManager(const Device& device_)
|
||||
: device{device_}, properties{device_.GetPhysical().GetMemoryProperties()} {}
|
||||
|
||||
VKMemoryManager::~VKMemoryManager() = default;
|
||||
|
||||
VKMemoryCommit VKMemoryManager::Commit(const VkMemoryRequirements& requirements,
|
||||
bool host_visible) {
|
||||
MemoryCommit VKMemoryManager::Commit(const VkMemoryRequirements& requirements, bool host_visible) {
|
||||
const u64 chunk_size = GetAllocationChunkSize(requirements.size);
|
||||
|
||||
// When a host visible commit is asked, search for host visible and coherent, otherwise search
|
||||
@ -131,39 +164,31 @@ VKMemoryCommit VKMemoryManager::Commit(const VkMemoryRequirements& requirements,
|
||||
const VkMemoryPropertyFlags wanted_properties =
|
||||
host_visible ? VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT
|
||||
: VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
|
||||
|
||||
if (auto commit = TryAllocCommit(requirements, wanted_properties)) {
|
||||
return commit;
|
||||
if (std::optional<MemoryCommit> commit = TryAllocCommit(requirements, wanted_properties)) {
|
||||
return std::move(*commit);
|
||||
}
|
||||
|
||||
// Commit has failed, allocate more memory.
|
||||
if (!AllocMemory(wanted_properties, requirements.memoryTypeBits, chunk_size)) {
|
||||
// TODO(Rodrigo): Handle these situations in some way like flushing to guest memory.
|
||||
// Allocation has failed, panic.
|
||||
UNREACHABLE_MSG("Ran out of VRAM!");
|
||||
return {};
|
||||
}
|
||||
// TODO(Rodrigo): Handle out of memory situations in some way like flushing to guest memory.
|
||||
AllocMemory(wanted_properties, requirements.memoryTypeBits, chunk_size);
|
||||
|
||||
// Commit again, this time it won't fail since there's a fresh allocation above. If it does,
|
||||
// there's a bug.
|
||||
auto commit = TryAllocCommit(requirements, wanted_properties);
|
||||
ASSERT(commit);
|
||||
return commit;
|
||||
// Commit again, this time it won't fail since there's a fresh allocation above.
|
||||
// If it does, there's a bug.
|
||||
return TryAllocCommit(requirements, wanted_properties).value();
|
||||
}
|
||||
|
||||
VKMemoryCommit VKMemoryManager::Commit(const vk::Buffer& buffer, bool host_visible) {
|
||||
MemoryCommit VKMemoryManager::Commit(const vk::Buffer& buffer, bool host_visible) {
|
||||
auto commit = Commit(device.GetLogical().GetBufferMemoryRequirements(*buffer), host_visible);
|
||||
buffer.BindMemory(commit->GetMemory(), commit->GetOffset());
|
||||
buffer.BindMemory(commit.Memory(), commit.Offset());
|
||||
return commit;
|
||||
}
|
||||
|
||||
VKMemoryCommit VKMemoryManager::Commit(const vk::Image& image, bool host_visible) {
|
||||
MemoryCommit VKMemoryManager::Commit(const vk::Image& image, bool host_visible) {
|
||||
auto commit = Commit(device.GetLogical().GetImageMemoryRequirements(*image), host_visible);
|
||||
image.BindMemory(commit->GetMemory(), commit->GetOffset());
|
||||
image.BindMemory(commit.Memory(), commit.Offset());
|
||||
return commit;
|
||||
}
|
||||
|
||||
bool VKMemoryManager::AllocMemory(VkMemoryPropertyFlags wanted_properties, u32 type_mask,
|
||||
void VKMemoryManager::AllocMemory(VkMemoryPropertyFlags wanted_properties, u32 type_mask,
|
||||
u64 size) {
|
||||
const u32 type = [&] {
|
||||
for (u32 type_index = 0; type_index < properties.memoryTypeCount; ++type_index) {
|
||||
@ -176,26 +201,18 @@ bool VKMemoryManager::AllocMemory(VkMemoryPropertyFlags wanted_properties, u32 t
|
||||
UNREACHABLE_MSG("Couldn't find a compatible memory type!");
|
||||
return 0U;
|
||||
}();
|
||||
|
||||
// Try to allocate found type.
|
||||
vk::DeviceMemory memory = device.GetLogical().TryAllocateMemory({
|
||||
vk::DeviceMemory memory = device.GetLogical().AllocateMemory({
|
||||
.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
|
||||
.pNext = nullptr,
|
||||
.allocationSize = size,
|
||||
.memoryTypeIndex = type,
|
||||
});
|
||||
if (!memory) {
|
||||
LOG_CRITICAL(Render_Vulkan, "Device allocation failed!");
|
||||
return false;
|
||||
}
|
||||
|
||||
allocations.push_back(std::make_unique<VKMemoryAllocation>(device, std::move(memory),
|
||||
allocations.push_back(std::make_unique<MemoryAllocation>(device, std::move(memory),
|
||||
wanted_properties, size, type));
|
||||
return true;
|
||||
}
|
||||
|
||||
VKMemoryCommit VKMemoryManager::TryAllocCommit(const VkMemoryRequirements& requirements,
|
||||
VkMemoryPropertyFlags wanted_properties) {
|
||||
std::optional<MemoryCommit> VKMemoryManager::TryAllocCommit(
|
||||
const VkMemoryRequirements& requirements, VkMemoryPropertyFlags wanted_properties) {
|
||||
for (auto& allocation : allocations) {
|
||||
if (!allocation->IsCompatible(wanted_properties, requirements.memoryTypeBits)) {
|
||||
continue;
|
||||
@ -204,27 +221,7 @@ VKMemoryCommit VKMemoryManager::TryAllocCommit(const VkMemoryRequirements& requi
|
||||
return commit;
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
VKMemoryCommitImpl::VKMemoryCommitImpl(const Device& device_, VKMemoryAllocation* allocation_,
|
||||
const vk::DeviceMemory& memory_, u64 begin_, u64 end_)
|
||||
: device{device_}, memory{memory_}, interval{begin_, end_}, allocation{allocation_} {}
|
||||
|
||||
VKMemoryCommitImpl::~VKMemoryCommitImpl() {
|
||||
allocation->Free(this);
|
||||
}
|
||||
|
||||
MemoryMap VKMemoryCommitImpl::Map(u64 size, u64 offset_) const {
|
||||
return MemoryMap(this, std::span<u8>(memory.Map(interval.first + offset_, size), size));
|
||||
}
|
||||
|
||||
void VKMemoryCommitImpl::Unmap() const {
|
||||
memory.Unmap();
|
||||
}
|
||||
|
||||
MemoryMap VKMemoryCommitImpl::Map() const {
|
||||
return Map(interval.second - interval.first);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
} // namespace Vulkan
|
||||
|
||||
@ -15,118 +15,81 @@ namespace Vulkan {
|
||||
|
||||
class Device;
|
||||
class MemoryMap;
|
||||
class VKMemoryAllocation;
|
||||
class VKMemoryCommitImpl;
|
||||
class MemoryAllocation;
|
||||
|
||||
using VKMemoryCommit = std::unique_ptr<VKMemoryCommitImpl>;
|
||||
|
||||
class VKMemoryManager final {
|
||||
class MemoryCommit final {
|
||||
public:
|
||||
explicit VKMemoryManager(const Device& device_);
|
||||
VKMemoryManager(const VKMemoryManager&) = delete;
|
||||
~VKMemoryManager();
|
||||
explicit MemoryCommit() noexcept = default;
|
||||
explicit MemoryCommit(const Device& device_, MemoryAllocation* allocation_,
|
||||
VkDeviceMemory memory_, u64 begin, u64 end) noexcept;
|
||||
~MemoryCommit();
|
||||
|
||||
/**
|
||||
* Commits a memory with the specified requeriments.
|
||||
* @param requirements Requirements returned from a Vulkan call.
|
||||
* @param host_visible Signals the allocator that it *must* use host visible and coherent
|
||||
* memory. When passing false, it will try to allocate device local memory.
|
||||
* @returns A memory commit.
|
||||
*/
|
||||
VKMemoryCommit Commit(const VkMemoryRequirements& requirements, bool host_visible);
|
||||
MemoryCommit& operator=(MemoryCommit&&) noexcept;
|
||||
MemoryCommit(MemoryCommit&&) noexcept;
|
||||
|
||||
/// Commits memory required by the buffer and binds it.
|
||||
VKMemoryCommit Commit(const vk::Buffer& buffer, bool host_visible);
|
||||
MemoryCommit& operator=(const MemoryCommit&) = delete;
|
||||
MemoryCommit(const MemoryCommit&) = delete;
|
||||
|
||||
/// Commits memory required by the image and binds it.
|
||||
VKMemoryCommit Commit(const vk::Image& image, bool host_visible);
|
||||
|
||||
private:
|
||||
/// Allocates a chunk of memory.
|
||||
bool AllocMemory(VkMemoryPropertyFlags wanted_properties, u32 type_mask, u64 size);
|
||||
|
||||
/// Tries to allocate a memory commit.
|
||||
VKMemoryCommit TryAllocCommit(const VkMemoryRequirements& requirements,
|
||||
VkMemoryPropertyFlags wanted_properties);
|
||||
|
||||
const Device& device; ///< Device handler.
|
||||
const VkPhysicalDeviceMemoryProperties properties; ///< Physical device properties.
|
||||
std::vector<std::unique_ptr<VKMemoryAllocation>> allocations; ///< Current allocations.
|
||||
};
|
||||
|
||||
class VKMemoryCommitImpl final {
|
||||
friend VKMemoryAllocation;
|
||||
friend MemoryMap;
|
||||
|
||||
public:
|
||||
explicit VKMemoryCommitImpl(const Device& device_, VKMemoryAllocation* allocation_,
|
||||
const vk::DeviceMemory& memory_, u64 begin_, u64 end_);
|
||||
~VKMemoryCommitImpl();
|
||||
|
||||
/// Maps a memory region and returns a pointer to it.
|
||||
/// It's illegal to have more than one memory map at the same time.
|
||||
MemoryMap Map(u64 size, u64 offset = 0) const;
|
||||
|
||||
/// Maps the whole commit and returns a pointer to it.
|
||||
/// It's illegal to have more than one memory map at the same time.
|
||||
MemoryMap Map() const;
|
||||
/// Returns a host visible memory map.
|
||||
/// It will map the backing allocation if it hasn't been mapped before.
|
||||
std::span<u8> Map();
|
||||
|
||||
/// Returns the Vulkan memory handler.
|
||||
VkDeviceMemory GetMemory() const {
|
||||
return *memory;
|
||||
VkDeviceMemory Memory() const {
|
||||
return memory;
|
||||
}
|
||||
|
||||
/// Returns the start position of the commit relative to the allocation.
|
||||
VkDeviceSize GetOffset() const {
|
||||
VkDeviceSize Offset() const {
|
||||
return static_cast<VkDeviceSize>(interval.first);
|
||||
}
|
||||
|
||||
private:
|
||||
/// Unmaps memory.
|
||||
void Unmap() const;
|
||||
void Release();
|
||||
|
||||
const Device& device; ///< Vulkan device.
|
||||
const vk::DeviceMemory& memory; ///< Vulkan device memory handler.
|
||||
const Device* device{}; ///< Vulkan device.
|
||||
MemoryAllocation* allocation{}; ///< Pointer to the large memory allocation.
|
||||
VkDeviceMemory memory{}; ///< Vulkan device memory handler.
|
||||
std::pair<u64, u64> interval{}; ///< Interval where the commit exists.
|
||||
VKMemoryAllocation* allocation{}; ///< Pointer to the large memory allocation.
|
||||
std::span<u8> span; ///< Host visible memory span. Empty if not queried before.
|
||||
};
|
||||
|
||||
/// Holds ownership of a memory map.
|
||||
class MemoryMap final {
|
||||
class VKMemoryManager final {
|
||||
public:
|
||||
explicit MemoryMap(const VKMemoryCommitImpl* commit_, std::span<u8> span_)
|
||||
: commit{commit_}, span{span_} {}
|
||||
explicit VKMemoryManager(const Device& device_);
|
||||
~VKMemoryManager();
|
||||
|
||||
~MemoryMap() {
|
||||
if (commit) {
|
||||
commit->Unmap();
|
||||
}
|
||||
}
|
||||
VKMemoryManager& operator=(const VKMemoryManager&) = delete;
|
||||
VKMemoryManager(const VKMemoryManager&) = delete;
|
||||
|
||||
/// Prematurely releases the memory map.
|
||||
void Release() {
|
||||
commit->Unmap();
|
||||
commit = nullptr;
|
||||
}
|
||||
/**
|
||||
* Commits a memory with the specified requeriments.
|
||||
*
|
||||
* @param requirements Requirements returned from a Vulkan call.
|
||||
* @param host_visible Signals the allocator that it *must* use host visible and coherent
|
||||
* memory. When passing false, it will try to allocate device local memory.
|
||||
*
|
||||
* @returns A memory commit.
|
||||
*/
|
||||
MemoryCommit Commit(const VkMemoryRequirements& requirements, bool host_visible);
|
||||
|
||||
/// Returns a span to the memory map.
|
||||
[[nodiscard]] std::span<u8> Span() const noexcept {
|
||||
return span;
|
||||
}
|
||||
/// Commits memory required by the buffer and binds it.
|
||||
MemoryCommit Commit(const vk::Buffer& buffer, bool host_visible);
|
||||
|
||||
/// Returns the address of the memory map.
|
||||
[[nodiscard]] u8* Address() const noexcept {
|
||||
return span.data();
|
||||
}
|
||||
|
||||
/// Returns the address of the memory map;
|
||||
[[nodiscard]] operator u8*() const noexcept {
|
||||
return span.data();
|
||||
}
|
||||
/// Commits memory required by the image and binds it.
|
||||
MemoryCommit Commit(const vk::Image& image, bool host_visible);
|
||||
|
||||
private:
|
||||
const VKMemoryCommitImpl* commit{}; ///< Mapped memory commit.
|
||||
std::span<u8> span; ///< Address to the mapped memory.
|
||||
/// Allocates a chunk of memory.
|
||||
void AllocMemory(VkMemoryPropertyFlags wanted_properties, u32 type_mask, u64 size);
|
||||
|
||||
/// Tries to allocate a memory commit.
|
||||
std::optional<MemoryCommit> TryAllocCommit(const VkMemoryRequirements& requirements,
|
||||
VkMemoryPropertyFlags wanted_properties);
|
||||
|
||||
const Device& device; ///< Device handler.
|
||||
const VkPhysicalDeviceMemoryProperties properties; ///< Physical device properties.
|
||||
std::vector<std::unique_ptr<MemoryAllocation>> allocations; ///< Current allocations.
|
||||
};
|
||||
|
||||
} // namespace Vulkan
|
||||
|
||||
@ -218,7 +218,7 @@ private:
|
||||
VKScheduler& scheduler;
|
||||
|
||||
VKStreamBuffer stream_buffer;
|
||||
VKStagingBufferPool staging_pool;
|
||||
StagingBufferPool staging_pool;
|
||||
VKDescriptorPool descriptor_pool;
|
||||
VKUpdateDescriptorQueue update_descriptor_queue;
|
||||
BlitImageHelper blit_image;
|
||||
@ -234,7 +234,7 @@ private:
|
||||
VKFenceManager fence_manager;
|
||||
|
||||
vk::Buffer default_buffer;
|
||||
VKMemoryCommit default_buffer_commit;
|
||||
MemoryCommit default_buffer_commit;
|
||||
vk::Event wfi_event;
|
||||
VideoCommon::Shader::AsyncShaders async_shaders;
|
||||
|
||||
|
||||
@ -3,58 +3,64 @@
|
||||
// Refer to the license.txt file included.
|
||||
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include "common/bit_util.h"
|
||||
#include "common/common_types.h"
|
||||
#include "video_core/renderer_vulkan/vk_scheduler.h"
|
||||
#include "video_core/renderer_vulkan/vk_staging_buffer_pool.h"
|
||||
#include "video_core/vulkan_common/vulkan_device.h"
|
||||
#include "video_core/vulkan_common/vulkan_wrapper.h"
|
||||
#include "video_core/vulkan_common/vulkan_device.h"
|
||||
|
||||
namespace Vulkan {
|
||||
|
||||
VKStagingBufferPool::StagingBuffer::StagingBuffer(std::unique_ptr<VKBuffer> buffer_)
|
||||
: buffer{std::move(buffer_)} {}
|
||||
|
||||
VKStagingBufferPool::VKStagingBufferPool(const Device& device_, VKMemoryManager& memory_manager_,
|
||||
StagingBufferPool::StagingBufferPool(const Device& device_, VKMemoryManager& memory_manager_,
|
||||
VKScheduler& scheduler_)
|
||||
: device{device_}, memory_manager{memory_manager_}, scheduler{scheduler_} {}
|
||||
|
||||
VKStagingBufferPool::~VKStagingBufferPool() = default;
|
||||
StagingBufferPool::~StagingBufferPool() = default;
|
||||
|
||||
VKBuffer& VKStagingBufferPool::GetUnusedBuffer(std::size_t size, bool host_visible) {
|
||||
if (const auto buffer = TryGetReservedBuffer(size, host_visible)) {
|
||||
return *buffer;
|
||||
StagingBufferRef StagingBufferPool::Request(size_t size, bool host_visible) {
|
||||
if (const std::optional<StagingBufferRef> ref = TryGetReservedBuffer(size, host_visible)) {
|
||||
return *ref;
|
||||
}
|
||||
return CreateStagingBuffer(size, host_visible);
|
||||
}
|
||||
|
||||
void VKStagingBufferPool::TickFrame() {
|
||||
current_delete_level = (current_delete_level + 1) % NumLevels;
|
||||
void StagingBufferPool::TickFrame() {
|
||||
current_delete_level = (current_delete_level + 1) % NUM_LEVELS;
|
||||
|
||||
ReleaseCache(true);
|
||||
ReleaseCache(false);
|
||||
}
|
||||
|
||||
VKBuffer* VKStagingBufferPool::TryGetReservedBuffer(std::size_t size, bool host_visible) {
|
||||
for (StagingBuffer& entry : GetCache(host_visible)[Common::Log2Ceil64(size)].entries) {
|
||||
if (!scheduler.IsFree(entry.tick)) {
|
||||
continue;
|
||||
std::optional<StagingBufferRef> StagingBufferPool::TryGetReservedBuffer(size_t size,
|
||||
bool host_visible) {
|
||||
StagingBuffers& cache_level = GetCache(host_visible)[Common::Log2Ceil64(size)];
|
||||
|
||||
const auto is_free = [this](const StagingBuffer& entry) {
|
||||
return scheduler.IsFree(entry.tick);
|
||||
};
|
||||
auto& entries = cache_level.entries;
|
||||
const auto hint_it = entries.begin() + cache_level.iterate_index;
|
||||
auto it = std::find_if(entries.begin() + cache_level.iterate_index, entries.end(), is_free);
|
||||
if (it == entries.end()) {
|
||||
it = std::find_if(entries.begin(), hint_it, is_free);
|
||||
if (it == hint_it) {
|
||||
return std::nullopt;
|
||||
}
|
||||
entry.tick = scheduler.CurrentTick();
|
||||
return &*entry.buffer;
|
||||
}
|
||||
return nullptr;
|
||||
cache_level.iterate_index = std::distance(entries.begin(), it) + 1;
|
||||
it->tick = scheduler.CurrentTick();
|
||||
return it->Ref();
|
||||
}
|
||||
|
||||
VKBuffer& VKStagingBufferPool::CreateStagingBuffer(std::size_t size, bool host_visible) {
|
||||
StagingBufferRef StagingBufferPool::CreateStagingBuffer(size_t size, bool host_visible) {
|
||||
const u32 log2 = Common::Log2Ceil64(size);
|
||||
|
||||
auto buffer = std::make_unique<VKBuffer>();
|
||||
buffer->handle = device.GetLogical().CreateBuffer({
|
||||
vk::Buffer buffer = device.GetLogical().CreateBuffer({
|
||||
.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
|
||||
.pNext = nullptr,
|
||||
.flags = 0,
|
||||
@ -66,49 +72,53 @@ VKBuffer& VKStagingBufferPool::CreateStagingBuffer(std::size_t size, bool host_v
|
||||
.queueFamilyIndexCount = 0,
|
||||
.pQueueFamilyIndices = nullptr,
|
||||
});
|
||||
buffer->commit = memory_manager.Commit(buffer->handle, host_visible);
|
||||
if (device.HasDebuggingToolAttached()) {
|
||||
++buffer_index;
|
||||
buffer.SetObjectNameEXT(fmt::format("Staging Buffer {}", buffer_index).c_str());
|
||||
}
|
||||
MemoryCommit commit = memory_manager.Commit(buffer, host_visible);
|
||||
const std::span<u8> mapped_span = host_visible ? commit.Map() : std::span<u8>{};
|
||||
|
||||
std::vector<StagingBuffer>& entries = GetCache(host_visible)[log2].entries;
|
||||
StagingBuffer& entry = entries.emplace_back(std::move(buffer));
|
||||
entry.tick = scheduler.CurrentTick();
|
||||
return *entry.buffer;
|
||||
StagingBuffer& entry = GetCache(host_visible)[log2].entries.emplace_back(StagingBuffer{
|
||||
.buffer = std::move(buffer),
|
||||
.commit = std::move(commit),
|
||||
.mapped_span = mapped_span,
|
||||
.tick = scheduler.CurrentTick(),
|
||||
});
|
||||
return entry.Ref();
|
||||
}
|
||||
|
||||
VKStagingBufferPool::StagingBuffersCache& VKStagingBufferPool::GetCache(bool host_visible) {
|
||||
StagingBufferPool::StagingBuffersCache& StagingBufferPool::GetCache(bool host_visible) {
|
||||
return host_visible ? host_staging_buffers : device_staging_buffers;
|
||||
}
|
||||
|
||||
void VKStagingBufferPool::ReleaseCache(bool host_visible) {
|
||||
auto& cache = GetCache(host_visible);
|
||||
const u64 size = ReleaseLevel(cache, current_delete_level);
|
||||
if (size == 0) {
|
||||
return;
|
||||
}
|
||||
void StagingBufferPool::ReleaseCache(bool host_visible) {
|
||||
ReleaseLevel(GetCache(host_visible), current_delete_level);
|
||||
}
|
||||
|
||||
u64 VKStagingBufferPool::ReleaseLevel(StagingBuffersCache& cache, std::size_t log2) {
|
||||
static constexpr std::size_t deletions_per_tick = 16;
|
||||
|
||||
void StagingBufferPool::ReleaseLevel(StagingBuffersCache& cache, size_t log2) {
|
||||
constexpr size_t deletions_per_tick = 16;
|
||||
auto& staging = cache[log2];
|
||||
auto& entries = staging.entries;
|
||||
const std::size_t old_size = entries.size();
|
||||
const size_t old_size = entries.size();
|
||||
|
||||
const auto is_deleteable = [this](const StagingBuffer& entry) {
|
||||
return scheduler.IsFree(entry.tick);
|
||||
};
|
||||
const std::size_t begin_offset = staging.delete_index;
|
||||
const std::size_t end_offset = std::min(begin_offset + deletions_per_tick, old_size);
|
||||
const auto begin = std::begin(entries) + begin_offset;
|
||||
const auto end = std::begin(entries) + end_offset;
|
||||
const size_t begin_offset = staging.delete_index;
|
||||
const size_t end_offset = std::min(begin_offset + deletions_per_tick, old_size);
|
||||
const auto begin = entries.begin() + begin_offset;
|
||||
const auto end = entries.begin() + end_offset;
|
||||
entries.erase(std::remove_if(begin, end, is_deleteable), end);
|
||||
|
||||
const std::size_t new_size = entries.size();
|
||||
const size_t new_size = entries.size();
|
||||
staging.delete_index += deletions_per_tick;
|
||||
if (staging.delete_index >= new_size) {
|
||||
staging.delete_index = 0;
|
||||
}
|
||||
|
||||
return (1ULL << log2) * (old_size - new_size);
|
||||
if (staging.iterate_index > new_size) {
|
||||
staging.iterate_index = 0;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Vulkan
|
||||
|
||||
@ -17,46 +17,54 @@ namespace Vulkan {
|
||||
class Device;
|
||||
class VKScheduler;
|
||||
|
||||
struct VKBuffer final {
|
||||
vk::Buffer handle;
|
||||
VKMemoryCommit commit;
|
||||
struct StagingBufferRef {
|
||||
VkBuffer buffer;
|
||||
std::span<u8> mapped_span;
|
||||
};
|
||||
|
||||
class VKStagingBufferPool final {
|
||||
class StagingBufferPool {
|
||||
public:
|
||||
explicit VKStagingBufferPool(const Device& device, VKMemoryManager& memory_manager,
|
||||
explicit StagingBufferPool(const Device& device, VKMemoryManager& memory_manager,
|
||||
VKScheduler& scheduler);
|
||||
~VKStagingBufferPool();
|
||||
~StagingBufferPool();
|
||||
|
||||
VKBuffer& GetUnusedBuffer(std::size_t size, bool host_visible);
|
||||
StagingBufferRef Request(size_t size, bool host_visible);
|
||||
|
||||
void TickFrame();
|
||||
|
||||
private:
|
||||
struct StagingBuffer final {
|
||||
explicit StagingBuffer(std::unique_ptr<VKBuffer> buffer);
|
||||
|
||||
std::unique_ptr<VKBuffer> buffer;
|
||||
struct StagingBuffer {
|
||||
vk::Buffer buffer;
|
||||
MemoryCommit commit;
|
||||
std::span<u8> mapped_span;
|
||||
u64 tick = 0;
|
||||
|
||||
StagingBufferRef Ref() const noexcept {
|
||||
return {
|
||||
.buffer = *buffer,
|
||||
.mapped_span = mapped_span,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
struct StagingBuffers final {
|
||||
struct StagingBuffers {
|
||||
std::vector<StagingBuffer> entries;
|
||||
std::size_t delete_index = 0;
|
||||
size_t delete_index = 0;
|
||||
size_t iterate_index = 0;
|
||||
};
|
||||
|
||||
static constexpr std::size_t NumLevels = sizeof(std::size_t) * CHAR_BIT;
|
||||
using StagingBuffersCache = std::array<StagingBuffers, NumLevels>;
|
||||
static constexpr size_t NUM_LEVELS = sizeof(size_t) * CHAR_BIT;
|
||||
using StagingBuffersCache = std::array<StagingBuffers, NUM_LEVELS>;
|
||||
|
||||
VKBuffer* TryGetReservedBuffer(std::size_t size, bool host_visible);
|
||||
std::optional<StagingBufferRef> TryGetReservedBuffer(size_t size, bool host_visible);
|
||||
|
||||
VKBuffer& CreateStagingBuffer(std::size_t size, bool host_visible);
|
||||
StagingBufferRef CreateStagingBuffer(size_t size, bool host_visible);
|
||||
|
||||
StagingBuffersCache& GetCache(bool host_visible);
|
||||
|
||||
void ReleaseCache(bool host_visible);
|
||||
|
||||
u64 ReleaseLevel(StagingBuffersCache& cache, std::size_t log2);
|
||||
void ReleaseLevel(StagingBuffersCache& cache, size_t log2);
|
||||
|
||||
const Device& device;
|
||||
VKMemoryManager& memory_manager;
|
||||
@ -65,7 +73,8 @@ private:
|
||||
StagingBuffersCache host_staging_buffers;
|
||||
StagingBuffersCache device_staging_buffers;
|
||||
|
||||
std::size_t current_delete_level = 0;
|
||||
size_t current_delete_level = 0;
|
||||
u64 buffer_index = 0;
|
||||
};
|
||||
|
||||
} // namespace Vulkan
|
||||
|
||||
@ -554,10 +554,10 @@ void TextureCacheRuntime::Finish() {
|
||||
}
|
||||
|
||||
ImageBufferMap TextureCacheRuntime::MapUploadBuffer(size_t size) {
|
||||
const auto& buffer = staging_buffer_pool.GetUnusedBuffer(size, true);
|
||||
const auto staging_ref = staging_buffer_pool.Request(size, true);
|
||||
return ImageBufferMap{
|
||||
.handle = *buffer.handle,
|
||||
.map = buffer.commit->Map(size),
|
||||
.handle = staging_ref.buffer,
|
||||
.span = staging_ref.mapped_span,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -19,14 +19,13 @@ using VideoCommon::Offset2D;
|
||||
using VideoCommon::RenderTargets;
|
||||
using VideoCore::Surface::PixelFormat;
|
||||
|
||||
class VKScheduler;
|
||||
class VKStagingBufferPool;
|
||||
|
||||
class BlitImageHelper;
|
||||
class Device;
|
||||
class Image;
|
||||
class ImageView;
|
||||
class Framebuffer;
|
||||
class StagingBufferPool;
|
||||
class VKScheduler;
|
||||
|
||||
struct RenderPassKey {
|
||||
constexpr auto operator<=>(const RenderPassKey&) const noexcept = default;
|
||||
@ -60,18 +59,18 @@ struct ImageBufferMap {
|
||||
}
|
||||
|
||||
[[nodiscard]] std::span<u8> Span() const noexcept {
|
||||
return map.Span();
|
||||
return span;
|
||||
}
|
||||
|
||||
VkBuffer handle;
|
||||
MemoryMap map;
|
||||
std::span<u8> span;
|
||||
};
|
||||
|
||||
struct TextureCacheRuntime {
|
||||
const Device& device;
|
||||
VKScheduler& scheduler;
|
||||
VKMemoryManager& memory_manager;
|
||||
VKStagingBufferPool& staging_buffer_pool;
|
||||
StagingBufferPool& staging_buffer_pool;
|
||||
BlitImageHelper& blit_image_helper;
|
||||
std::unordered_map<RenderPassKey, vk::RenderPass> renderpass_cache;
|
||||
|
||||
@ -141,7 +140,7 @@ private:
|
||||
VKScheduler* scheduler;
|
||||
vk::Image image;
|
||||
vk::Buffer buffer;
|
||||
VKMemoryCommit commit;
|
||||
MemoryCommit commit;
|
||||
VkImageAspectFlags aspect_mask = 0;
|
||||
bool initialized = false;
|
||||
};
|
||||
|
||||
Loading…
Reference in New Issue
Block a user