device.c

#include <kiba/containers/array.h>
#include <kiba/core/memory.h>
#include <kiba/core/string.h>
#include <kiba/gpu/vulkan/allocator.h>
#include <kiba/gpu/vulkan/conv.h>
#include <kiba/gpu/vulkan/device.h>
#include <kiba/gpu/vulkan/instance.h>
#include <kiba/gpu/vulkan/queue.h>
#include <kiba/gpu/vulkan/util.h>
 
// FIXME ideally the memory header should include a memcmp equivalent
#include <string.h>
 
// TODO ideally we'd probably want to get all devices as part of the vulkan initialization
// then just hand out the devices based on requirements. that should also simplify queue sharing
 
struct vk_device_requirements {
    b8                       graphics;
    b8                       transfer;
    b8                       present;
    b8                       compute;
    b8                       discrete_gpu;
    VkPhysicalDeviceFeatures features;
    array_of(const char *) extensions;
};
 
b8 vk_select_physical_device(struct gpu_backend_device *device, struct vk_device_requirements requirements);
b8 vk_select_logical_device(struct gpu_backend_device *device, struct vk_device_requirements requirements);
 
b8 vk_physical_device_meets_requirements(struct gpu_backend_device    *device,
                                         VkPhysicalDevice              phys_device,
                                         VkPhysicalDeviceFeatures     *features,
                                         VkPhysicalDeviceProperties   *props,
                                         struct vk_device_requirements requirements);
 
b8 gpu_backend_device_create(struct gpu_backend_device *device) {
    // TODO temp default config
    array_of(const char *) device_extensions = array_create(const char *, 4, &vk_alloc.kiba_alloc);
    if (device_extensions == KB_NULL) {
        KB_ERROR("could not create array for device extensions");
        return false;
    }
    array_push(device_extensions, VK_KHR_SWAPCHAIN_EXTENSION_NAME);
    array_push(device_extensions, VK_KHR_SHADER_NON_SEMANTIC_INFO_EXTENSION_NAME);
    VkPhysicalDeviceFeatures features = {
        .samplerAnisotropy = VK_TRUE,
        .fillModeNonSolid  = VK_TRUE,
    };
    struct vk_device_requirements requirements = {
        .graphics     = true,
        .transfer     = true,
        .present      = true,
        .compute      = true,
        .discrete_gpu = true,
        .extensions   = device_extensions,
        .features     = features,
    };
 
    if (!vk_select_physical_device(device, requirements)) {
        KB_ERROR("failed to find GPU meeting the requirements");
        return false;
    }
    if (!vk_select_logical_device(device, requirements)) {
        KB_ERROR("failed to create logical device");
        return false;
    }
    device->framebuffers = array_create(struct vk_framebuffer, 8, &vk_alloc.kiba_alloc);
    if (device->framebuffers == KB_NULL) {
        KB_ERROR("cannot reserve memory for framebuffer storage");
        return false;
    }
    device->renderpasses = array_create(struct vk_renderpass, 8, &vk_alloc.kiba_alloc);
    if (device->renderpasses == KB_NULL) {
        KB_ERROR("cannot reserve memory for renderpass storage");
        return false;
    }
 
    return true;
}
 
void gpu_backend_device_finish_running_tasks(struct gpu_backend_device *device) { vkDeviceWaitIdle(device->logical); }
 
void gpu_backend_device_destroy(struct gpu_backend_device *device) {
    vk_queue_destroy(device->logical, &device->graphics_queue);
    vk_queue_destroy(device->logical, &device->transfer_queue);
    vk_queue_destroy(device->logical, &device->present_queue);
    vk_queue_destroy(device->logical, &device->compute_queue);
    array_for_each(struct vk_framebuffer, fb, device->framebuffers) {
        vkDestroyFramebuffer(device->logical, fb->fb, &vk_alloc.vulkan_callbacks);
    }
    array_destroy(&device->framebuffers);
    array_for_each(struct vk_renderpass, rp, device->renderpasses) {
        vkDestroyRenderPass(device->logical, rp->rp, &vk_alloc.vulkan_callbacks);
    }
    array_destroy(&device->renderpasses);
    if (device->logical != VK_NULL_HANDLE) {
        vkDestroyDevice(device->logical, &vk_alloc.vulkan_callbacks);
    }
    memory_zero(device, sizeof(struct gpu_backend_device));
}
 
b8 vk_select_physical_device(struct gpu_backend_device *device, struct vk_device_requirements requirements) {
    u32 physical_device_count = 0;
    VK_CALL_B8(vkEnumeratePhysicalDevices(vk_instance.raw, &physical_device_count, 0));
    if (physical_device_count == 0) {
        KB_INFO("found no device that supports vulkan");
        return false;
    }
 
    array_of(VkPhysicalDevice) physical_devices =
        array_create(VkPhysicalDevice, physical_device_count, &vk_alloc.kiba_alloc);
    array_resize(&physical_devices, physical_device_count);
    if (physical_devices == KB_NULL) {
        KB_ERROR("unable to allocate enough memory for list of physical devices");
        return false;
    }
    VK_CALL_B8(vkEnumeratePhysicalDevices(vk_instance.raw, &physical_device_count, physical_devices));
 
    b8 ret = false;
    array_for_each(const VkPhysicalDevice, physical_device, physical_devices) {
        VkPhysicalDeviceProperties physical_device_properties;
        VkPhysicalDeviceFeatures   physical_device_features;
        vkGetPhysicalDeviceProperties(*physical_device, &physical_device_properties);
        vkGetPhysicalDeviceFeatures(*physical_device, &physical_device_features);
        KB_INFO("checking device {raw_string} against requirements", physical_device_properties.deviceName);
        if (vk_physical_device_meets_requirements(device,
                                                  *physical_device,
                                                  &physical_device_features,
                                                  &physical_device_properties,
                                                  requirements)) {
            KB_INFO("device {raw_string} meets requirements", physical_device_properties.deviceName);
            device->physical         = *physical_device;
            ret                      = true;
            break;
        }
        KB_INFO("skipping physical device {raw_string} because it does not match the requirements",
                physical_device_properties.deviceName);
    }
    array_destroy(&physical_devices);
    return ret;
}
 
b8 vk_select_logical_device(struct gpu_backend_device *device, struct vk_device_requirements requirements) {
    const usize max_different_queue = 8;
 
    u32 indices[max_different_queue];
    u32 index        = 0;
    indices[index++] = device->graphics_queue.index;
    if (device->graphics_queue.index != device->present_queue.index) {
        indices[index++] = device->present_queue.index;
    }
    if (device->graphics_queue.index != device->transfer_queue.index) {
        indices[index++] = device->transfer_queue.index;
    }
 
    f32                     high_prio = 1.0f;
    VkDeviceQueueCreateInfo queue_create_infos[max_different_queue];
    for (u32 i = 0; i < index; ++i) {
        queue_create_infos[i].sType            = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
        queue_create_infos[i].queueFamilyIndex = indices[i];
        // TODO: should be configurable how many
        queue_create_infos[i].queueCount       = 1;
        queue_create_infos[i].flags            = 0;
        queue_create_infos[i].pNext            = 0;
        queue_create_infos[i].pQueuePriorities = &high_prio;
    }
 
    VkDeviceCreateInfo device_create_info = {
        .sType                   = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
        .pQueueCreateInfos       = queue_create_infos,
        .queueCreateInfoCount    = index,
        .pEnabledFeatures        = &requirements.features,
        .ppEnabledExtensionNames = requirements.extensions,
        .enabledExtensionCount   = (u32) array_size(requirements.extensions),
    };
    VK_CALL_B8(vkCreateDevice(device->physical, &device_create_info, &vk_alloc.vulkan_callbacks, &device->logical));
 
    if (!vk_queue_create(device->logical, &device->graphics_queue)) {
        KB_ERROR("failed to initialize graphics queue");
        device->graphics_queue.available = false;
        return false;
    }
    if (!vk_queue_create(device->logical, &device->transfer_queue)) {
        KB_ERROR("failed to initialize transfer queue");
        device->transfer_queue.available = false;
        return false;
    }
    if (!vk_queue_create(device->logical, &device->present_queue)) {
        KB_ERROR("failed to initialize present queue");
        device->present_queue.available = false;
        return false;
    }
    if (!vk_queue_create(device->logical, &device->compute_queue)) {
        KB_ERROR("failed to initialize compute queue");
        device->compute_queue.available = false;
        return false;
    }
    return true;
}
 
b8 vk_physical_device_meets_requirements(struct gpu_backend_device    *device,
                                         VkPhysicalDevice              phys_device,
                                         VkPhysicalDeviceFeatures     *features,
                                         VkPhysicalDeviceProperties   *props,
                                         struct vk_device_requirements requirements) {
    UNUSED(features);
    // discrete gpu requirement
    if (requirements.discrete_gpu && props->deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU) {
        KB_INFO("device is not viable as its not a discrete GPU");
        return false;
    }
 
    // queue families
    struct vk_queue graphics_queue;
    struct vk_queue transfer_queue;
    struct vk_queue present_queue;
    struct vk_queue compute_queue;
 
    u32 queue_family_count = 0;
    vkGetPhysicalDeviceQueueFamilyProperties(phys_device, &queue_family_count, 0);
 
    array_of(VkQueueFamilyProperties) queue_props =
        array_create(VkQueueFamilyProperties, queue_family_count, &vk_alloc.kiba_alloc);
    array_resize(&queue_props, queue_family_count);
    if (queue_props == KB_NULL) {
        KB_INFO("unable to allocate enough memory for list of queue properties");
        return false;
    }
    vkGetPhysicalDeviceQueueFamilyProperties(phys_device, &queue_family_count, queue_props);
 
    for (u32 i = 0; i < queue_family_count; ++i) {
        if (queue_props[i].queueFlags & VK_QUEUE_GRAPHICS_BIT && !device->graphics_queue.available) {
            graphics_queue.available = true;
            graphics_queue.index     = i;
            VkBool32 present_support = true;  // false;
            // TODO reintroduce this somehow? maybe it's not even needed
            // VK_CALL_B8(vkGetPhysicalDeviceSurfaceSupportKHR(device, i, context->surface, &present_support));
            if (present_support) {
                present_queue.available = true;
                present_queue.index     = i;
            }
        }
        if (queue_props[i].queueFlags & VK_QUEUE_TRANSFER_BIT) {
            transfer_queue.available = true;
            transfer_queue.index     = i;
        }
        if (queue_props[i].queueFlags & VK_QUEUE_COMPUTE_BIT) {
            compute_queue.available = true;
            compute_queue.index     = i;
        }
    }
    array_destroy(&queue_props);
    if (requirements.graphics && !graphics_queue.available) {
        KB_INFO("device does not support required graphics queue");
        return false;
    }
    if (requirements.transfer && !transfer_queue.available) {
        KB_INFO("device does not support required transfer queue");
        return false;
    }
    if (requirements.present && !present_queue.available) {
        KB_INFO("device does not support required present queue");
        return false;
    }
    if (requirements.compute && !compute_queue.available) {
        KB_INFO("device does not support required compute queue");
        return false;
    }
 
    // device extensions
    u32 extension_count = 0;
    vkEnumerateDeviceExtensionProperties(phys_device, 0, &extension_count, 0);
 
    array_of(VkExtensionProperties) extensions =
        array_create(VkExtensionProperties, extension_count, &vk_alloc.kiba_alloc);
    array_resize(&extensions, extension_count);
    if (extensions == KB_NULL) {
        KB_INFO("unable to allocate enough memory for list of extensions");
        return false;
    }
    vkEnumerateDeviceExtensionProperties(phys_device, 0, &extension_count, extensions);
 
    u32 required_extension_count = (u32) array_size(requirements.extensions);
    b8  has_extensions           = true;
    if (extension_count >= required_extension_count) {
        array_for_each(const char *, extension, requirements.extensions) {
            b8 found = false;
            for (u32 j = 0; j < extension_count; ++j) {
                if (string_equal(string_from_raw(*extension), string_from_raw(extensions[j].extensionName))) {
                    found = true;
                    break;
                }
            }
            if (!found) {
                KB_INFO("required extension {raw_string} not supported by device", *extension);
                has_extensions = false;
            }
        }
    } else {
        KB_INFO("device cannot support all required extensions");
        has_extensions = false;
    }
 
    array_destroy(&extensions);
 
    device->graphics_queue = graphics_queue;
    device->transfer_queue = transfer_queue;
    device->present_queue  = present_queue;
    device->compute_queue  = compute_queue;
 
    return has_extensions;
}
 
static u32 vk_device_get_memory_type(struct gpu_backend_device device,
                                     u32                       type_filter,
                                     VkMemoryPropertyFlagBits  flags) {
    VkPhysicalDeviceMemoryProperties memory_properties;
    vkGetPhysicalDeviceMemoryProperties(device.physical, &memory_properties);
    for (u32 i = 0; i < memory_properties.memoryTypeCount; ++i) {
        if (type_filter & KB_UBIT(i) && KB_FLAGS_ALL_SET(memory_properties.memoryTypes[i].propertyFlags, flags)) {
            return i;
        }
    }
    return U32_MAX;
}
 
b8 vk_device_allocate_memory(struct gpu_backend_device device,
                             VkDeviceMemory           *memory,
                             VkMemoryRequirements      requirements,
                             VkMemoryPropertyFlagBits  properties) {
    u32 memory_type_index = vk_device_get_memory_type(device, requirements.memoryTypeBits, properties);
    KB_ASSERT(memory_type_index != U32_MAX,
              "memory type must be available");  // TODO should have fallback handling
    VkMemoryAllocateInfo alloc_info = {
        .sType           = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
        .allocationSize  = requirements.size,
        .memoryTypeIndex = memory_type_index,
    };
    VK_CALL_B8(vkAllocateMemory(device.logical, &alloc_info, &vk_alloc.vulkan_callbacks, memory));
    return true;
}
 
b8 vk_device_create_framebuffer(struct gpu_backend_device *device,
                                struct vk_framebuffer_key  key,
                                VkRenderPass               renderpass,
                                VkFramebuffer             *out_framebuffer) {
    array_for_each(struct vk_framebuffer, fb, device->framebuffers) {
        if (memcmp(&fb->key, &key, sizeof(struct vk_framebuffer_key)) == 0) {
            *out_framebuffer = fb->fb;
            KB_DEBUG("serving framebuffer from cache");
            return true;
        }
    }
    KB_DEBUG("creating framebuffer");
    VkImageView attachments[KB_GPU_MAX_ATTACHMENTS] = {0};
    for (u32 i = 0; i < key.attachment_count; ++i) {
        attachments[i] = key.attachments[i].view;
    }
    VkFramebufferCreateInfo framebuffer_info = {
        .sType           = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
        .renderPass      = renderpass,
        .attachmentCount = key.attachment_count,
        .pAttachments    = attachments,
        .width           = key.extent.width,
        .height          = key.extent.height,
        .layers          = key.extent.depth ? key.extent.depth : 1,  // TODO keep the safeguard against zero?
    };
    VK_CALL_B8(vkCreateFramebuffer(device->logical, &framebuffer_info, &vk_alloc.vulkan_callbacks, out_framebuffer));
    struct vk_framebuffer new_fb = {
        .key = key,
        .fb  = *out_framebuffer,
    };
    return array_push_checked(&device->framebuffers, &new_fb);
}
 
b8 vk_device_create_renderpass(struct gpu_backend_device *device,
                               struct vk_renderpass_key   key,
                               VkRenderPass              *out_renderpass) {
    array_for_each(struct vk_renderpass, rp, device->renderpasses) {
        if (memcmp(&rp->key, &key, sizeof(struct vk_renderpass_key)) == 0) {
            *out_renderpass = rp->rp;
            KB_DEBUG("serving render pass from cache");
            return true;
        }
    }
    KB_DEBUG("creating render pass");
    u32                     attachment_descr_count                     = 0;
    VkAttachmentDescription attachment_descr[KB_GPU_MAX_ATTACHMENTS]   = {0};
    u32                     color_ref_count                            = 0;
    VkAttachmentReference   color_refs[KB_GPU_MAX_COLOR_ATTACHMENTS]   = {0};
    u32                     resolve_ref_count                          = 0;
    VkAttachmentReference   resolve_refs[KB_GPU_MAX_COLOR_ATTACHMENTS] = {0};
 
    VkAttachmentReference unused_ref = {
        .attachment = VK_ATTACHMENT_UNUSED,
        .layout     = VK_IMAGE_LAYOUT_UNDEFINED,
    };
    for (u32 ca = 0; ca < key.color_count; ++ca) {
        const struct vk_color_attachment_key color_attachment = key.colors[ca];
        VkAttachmentReference                color_ref        = {
                                  .attachment = attachment_descr_count,
                                  .layout     = color_attachment.base.layout,
        };
        color_refs[color_ref_count++]       = color_ref;
        VkAttachmentDescription color_descr = {
            .format        = color_attachment.base.format,
            .samples       = VK_SAMPLE_COUNT_1_BIT,
            .loadOp        = vk_convert_attachment_load_op(color_attachment.base.ops),
            .storeOp       = vk_convert_attachment_store_op(color_attachment.base.ops),
            .initialLayout = color_attachment.base.layout,
            .finalLayout   = color_attachment.base.layout,
        };
        attachment_descr[attachment_descr_count++] = color_descr;
 
        VkAttachmentReference resolve_ref = unused_ref;
        if (color_attachment.resolve_set) {
            resolve_ref = (VkAttachmentReference){
                .attachment = attachment_descr_count,
                .layout     = color_attachment.resolve.layout,
            };
            VkAttachmentDescription resolve_descr = {
                .format        = color_attachment.resolve.format,
                .samples       = VK_SAMPLE_COUNT_1_BIT,
                .loadOp        = vk_convert_attachment_load_op(color_attachment.resolve.ops),
                .storeOp       = vk_convert_attachment_store_op(color_attachment.resolve.ops),
                .initialLayout = color_attachment.resolve.layout,
                .finalLayout   = color_attachment.resolve.layout,
            };
            attachment_descr[attachment_descr_count++] = resolve_descr;
        }
        resolve_refs[resolve_ref_count++] = resolve_ref;
    }
 
    VkSubpassDescription subpass = {
        .pipelineBindPoint    = VK_PIPELINE_BIND_POINT_GRAPHICS,
        .colorAttachmentCount = color_ref_count,
        .pColorAttachments    = color_refs,
        .pResolveAttachments  = resolve_refs,
    };
 
    VkAttachmentReference depth_ref;
    b8                    set_depth_ref = false;
    if (key.depth_stencil_set) {
        set_depth_ref = true;
        depth_ref     = (VkAttachmentReference){
                .attachment = attachment_descr_count,
                .layout     = key.depth_stencil.base.layout,
        };
        VkAttachmentDescription depth_descr = {
            .format         = key.depth_stencil.base.format,
            .samples        = VK_SAMPLE_COUNT_1_BIT,
            .loadOp         = vk_convert_attachment_load_op(key.depth_stencil.base.ops),
            .storeOp        = vk_convert_attachment_store_op(key.depth_stencil.base.ops),
            .stencilLoadOp  = vk_convert_attachment_load_op(key.depth_stencil.stencil_ops),
            .stencilStoreOp = vk_convert_attachment_store_op(key.depth_stencil.stencil_ops),
            .initialLayout  = VK_IMAGE_LAYOUT_UNDEFINED,  // TODO key.depth_stencil.base.layout,
            .finalLayout    = key.depth_stencil.base.layout,
        };
        attachment_descr[attachment_descr_count++] = depth_descr;
    }
    subpass.pDepthStencilAttachment = set_depth_ref ? &depth_ref : NULL;
 
    VkSubpassDependency subpass_dependency = {
        // TODO is this still needed? -> technically not but may be useful in other use cases to automatically
        // transition swapchain images
        .srcSubpass    = VK_SUBPASS_EXTERNAL,
        .dstSubpass    = 0,
        .srcStageMask  = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT,
        .srcAccessMask = 0,
        .dstStageMask  = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT,
        .dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT,
    };
 
    VkRenderPassCreateInfo renderpass_info = {
        .sType           = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
        .attachmentCount = attachment_descr_count,
        .pAttachments    = attachment_descr,
        .subpassCount    = 1,
        .pSubpasses      = &subpass,
        .dependencyCount = 1,
        .pDependencies   = &subpass_dependency,
    };
 
    VK_CALL_B8(vkCreateRenderPass(device->logical, &renderpass_info, &vk_alloc.vulkan_callbacks, out_renderpass));
    struct vk_renderpass new_rp = {
        .key = key,
        .rp  = *out_renderpass,
    };
    return array_push_checked(&device->renderpasses, &new_rp);
}
 
void vk_device_set_object_name(VkDevice device, VkObjectType type, void *handle, const char *name) {
#ifdef KB_DEBUG_BUILD
    VkDebugUtilsObjectNameInfoEXT info = {
        .sType        = VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT,
        .objectType   = type,
        .objectHandle = (u64) handle,
        .pObjectName  = name,
    };
    VkResult res = vk_instance.setDebugUtilsObjectName(device, &info);
    if (res != VK_SUCCESS) {
        KB_WARN("error setting object name {raw_string} for object {pointer} of type {u32}: {u32}",
                name,
                handle,
                type,
                res);
    }
#else
    UNUSED(device);
    UNUSED(type);
    UNUSED(handle);
    UNUSED(name);
#endif
}