From 02076a41dc1764487b68b840ce5c61952801c5e3 Mon Sep 17 00:00:00 2001 From: Jakub Dabek Date: Mon, 16 Jan 2023 20:07:52 +0100 Subject: [PATCH] memory management: Add virtual heap memory allocators Add virtual heap allocators that allocate on proper heaps and map physical memory where nesscessary. Add free function that frees up the virtual heap memory and unmaps physical memory when possible. Virtual heap allocator allows using virtual memory as a base for allocation and booking memory. Physical memory banks will be mapped when needed allowing for greater flexibility with mapping. Signed-off-by: Jakub Dabek --- zephyr/include/sof/lib/regions_mm.h | 133 ++++- zephyr/lib/regions_mm.c | 740 +++++++++++++++++++++++++++- 2 files changed, 826 insertions(+), 47 deletions(-) diff --git a/zephyr/include/sof/lib/regions_mm.h b/zephyr/include/sof/lib/regions_mm.h index 2e52cdcabf67..7e9d3a1a40a7 100644 --- a/zephyr/include/sof/lib/regions_mm.h +++ b/zephyr/include/sof/lib/regions_mm.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause */ /* - * Copyright(c) 2022 Intel Corporation. All rights reserved. + * Copyright(c) 2022 - 2023 Intel Corporation. All rights reserved. * * Author: Jakub Dabek */ @@ -10,31 +10,124 @@ #include #include +#include +#include +#include #include -#include #include -#include +#include + +/* Dependency on ipc/topology.h created due to memory capability definitions + * that are defined there + */ + +/* API is NOT re-entry safe. + * This is due to our assumption that only management code will ever handle + * memory operations on heaps themselves. + */ + +/* Defines maximum amount of memory block allocators in one heap + * since minimum realistic block should be cache line size + * and block sizes in allocator must be powers of 2 + * so logically it gives limited block sizes + * eg allocator of block size 64 128 256 512 2048 4096 1024 8192 + * would end up in 8 allocators. + * It is expected that allocations bigger than that would + * either be spanned on specifically configured heap or have + * individual configs with bigger block sizes. + */ +#define MAX_MEMORY_ALLOCATORS_COUNT 8 + +/* vmh_get_default_heap_config() function will try to split the region + * down the given count. Only applicable when API client did not + * use its config. + */ +#define DEFAULT_CONFIG_ALOCATORS_COUNT 5 + +/** @struct vmh_heap + * + * @brief This structure holds all information about virtual memory heap + * it aggregates information about its allocations and + * physical mappings. + * + * @var node generic list member used for list operations + * @var virtual_region pointer to virtual region information, it holds its + * attributes size and beginning ptr provided by zephyr. + * @var physical_blocks_allocators[] a table of block allocators + * each representing a virtual regions part in blocks of a given size + * governed by sys_mem_blocks API. + * @var allocation_sizes[] a table of bit arrays representing sizes of allocations + * made in physical_blocks_allocators directly related to physical_blocks_allocators + * @var core_id id of the core that heap was created on + * @var allocating_continuously configuration value deciding if heap allocations + * will be contiguous or single block. + */ +struct vmh_heap { + struct list_item node; + const struct sys_mm_drv_region *virtual_region; + struct sys_mem_blocks *physical_blocks_allocators[MAX_MEMORY_ALLOCATORS_COUNT]; + struct sys_bitarray *allocation_sizes[MAX_MEMORY_ALLOCATORS_COUNT]; + int core_id; + bool allocating_continuously; +}; + +/** @struct vmh_block_bundle_descriptor + * + * @brief This is a struct describing one bundle of blocks + * used as base for allocators blocks. + * + * @var block_size size of memory block. + * @var number_of_blocks number of memory blocks. + */ +struct vmh_block_bundle_descriptor { + size_t block_size; + size_t number_of_blocks; +}; /* - * Struct containing information on virtual memory heap. - * Information about allocated physical blocks is stored in - * physical_blocks_allocators variable and uses zephyr memblocks api. - */ -struct virtual_memory_heap { - /* zephyr provided virtual region */ - struct sys_mm_drv_region *virtual_region; - /* physical pages allocators represented in memblocks */ - struct sys_multi_mem_blocks physical_blocks_allocators; - /* SOF memory capability */ - uint32_t memory_caps; + * Maybe this heap config should have small bundles first going up to max + * size or there should be a sorting mechanism for those ? + * Should we assume that bundles are given from smallest to biggest ? + */ + +/** @struct vmh_heap_config + * + * @brief This is a struct that aggregates vmh_block_bundle_descriptor to + * create one cfg that can be passed to heap initiation. + * Provided config size must be physical page aligned so it + * will not overlap in physical space with other heaps during mapping. + * So every block has to have its overall size aligned to CONFIG_MM_DRV_PAGE_SIZE + * + * @vmh_block_bundle_descriptor[] aggregation of bundle descriptors. + */ +struct vmh_heap_config { + struct vmh_block_bundle_descriptor block_bundles_table[MAX_MEMORY_ALLOCATORS_COUNT]; }; -/* Available externaly array containing all information on virtual heaps - * Used to control physical allocations and overall virtual to physicall - * mapping on sof side (zephyr handles the actual assigning physical memory - * sof only requests it). +struct vmh_heap *vmh_init_heap(const struct vmh_heap_config *cfg, + int memory_region_attribute, int core_id, bool allocating_continuously); +void *vmh_alloc(struct vmh_heap *heap, uint32_t alloc_size); +int vmh_free_heap(struct vmh_heap *heap); +int vmh_free(struct vmh_heap *heap, void *ptr); +struct vmh_heap *vmh_reconfigure_heap(struct vmh_heap *heap, + struct vmh_heap_config *cfg, int core_id, bool allocating_continuously); +void vmh_get_default_heap_config(const struct sys_mm_drv_region *region, + struct vmh_heap_config *cfg); +struct vmh_heap *vmh_get_heap_by_attribute(uint32_t attr, uint32_t core_id); +/** + * @brief Checks if ptr is in range of given memory range + * + * @param ptr checked ptr + * @param range_start start of checked memory range + * @param range_size size of checked memory range. + * + * @retval false if no overlap detected. + * @retval true if any overlap detected. */ -extern struct virtual_memory_heap - vm_heaps[CONFIG_MP_MAX_NUM_CPUS + VIRTUAL_REGION_COUNT]; +static inline bool vmh_is_ptr_in_memory_range(uintptr_t ptr, uintptr_t range_start, + size_t range_size) +{ + return ptr >= range_start && ptr < range_start + range_size; +} #endif /* ZEPHYR_LIB_REGIONS_MM_H_ */ diff --git a/zephyr/lib/regions_mm.c b/zephyr/lib/regions_mm.c index 7921c3c020d3..4e3356f7fd98 100644 --- a/zephyr/lib/regions_mm.c +++ b/zephyr/lib/regions_mm.c @@ -1,52 +1,738 @@ // SPDX-License-Identifier: BSD-3-Clause /* - * Copyright(c) 2022 Intel Corporation. All rights reserved. + * Copyright(c) 2022 - 2023 Intel Corporation. All rights reserved. * * Author: Jakub Dabek */ #include -struct virtual_memory_heap - vm_heaps[CONFIG_MP_MAX_NUM_CPUS + VIRTUAL_REGION_COUNT]; +/* list of vmh_heap objects created */ +static struct list_item vmh_list; /** - * @brief Fill vm_heaps array with information from zephyr. + * @brief Initialize new heap * - * Virtual memory regions calculated in zephyr are translated here - * to a struct that will keep all information on current allocations - * and virtual to physical mappings that are related to heaps. - * System heap is not a part of this information. It only refers to - * virtual first heaps. - * Has to be initialized after calculations for regions is done in zephyr. + * Initializes new heap based on region that heap is to be assigned to + * and config. The heap size overall must be aligned to physical page + * size. + * @param cfg Configuration structure with block structure for the heap + * @param memory_region_attribute A zephyr defined virtual region identifier + * @param core_id Core id of a heap that will be created + * @param allocating_continuously Flag deciding if heap can do contiguous allocation. + * + * @retval ptr to a new heap if successful. + * @retval NULL on creation failure. */ -static int virtual_heaps_init(void) +struct vmh_heap *vmh_init_heap(const struct vmh_heap_config *cfg, + int memory_region_attribute, int core_id, bool allocating_continuously) { - struct sys_mm_drv_region *virtual_memory_regions = - (struct sys_mm_drv_region *)sys_mm_drv_query_memory_regions(); + const struct sys_mm_drv_region *virtual_memory_regions = + sys_mm_drv_query_memory_regions(); + int i; - for (size_t i = 0; - i < CONFIG_MP_MAX_NUM_CPUS + VIRTUAL_REGION_COUNT; - i++) { - vm_heaps[i].virtual_region = &virtual_memory_regions[i]; + /* Check if we haven't created heap for this region already */ + if (vmh_get_heap_by_attribute(memory_region_attribute, core_id)) + return NULL; - switch (virtual_memory_regions[i].attr) { - case MEM_REG_ATTR_CORE_HEAP: - vm_heaps[i].memory_caps = - SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_HP | SOF_MEM_CAPS_CACHE; - break; - case MEM_REG_ATTR_SHARED_HEAP: - vm_heaps[i].memory_caps = SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_HP; + struct vmh_heap *new_heap = + rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM, sizeof(*new_heap)); + + if (!new_heap) + return NULL; + + new_heap->core_id = core_id; + list_init(&new_heap->node); + struct vmh_heap_config new_config = {0}; + + /* Search for matching attribute so we place heap on right + * virtual region. + * In case of core heaps regions we count theme from 0 to max + * available cores + */ + if (memory_region_attribute == MEM_REG_ATTR_CORE_HEAP) { + new_heap->virtual_region = &virtual_memory_regions[core_id]; + } else { + for (i = CONFIG_MP_MAX_NUM_CPUS; + i < CONFIG_MP_MAX_NUM_CPUS + VIRTUAL_REGION_COUNT; i++) { + + if (memory_region_attribute == virtual_memory_regions[i].attr) { + new_heap->virtual_region = &virtual_memory_regions[i]; + break; + } + } + } + + if (!new_heap->virtual_region) + goto fail; + + /* If no config was specified we will use default one */ + if (!cfg) { + vmh_get_default_heap_config(new_heap->virtual_region, &new_config); + cfg = &new_config; + } + + /* Check if configuration provided can fit into virtual regions memory + * and if it is physical page aligned + */ + size_t total_requested_size = 0; + + for (i = 0; i < MAX_MEMORY_ALLOCATORS_COUNT; i++) { + if (cfg->block_bundles_table[i].block_size > 0) { + /* Block sizes can only be powers of 2*/ + if (!is_power_of_2(cfg->block_bundles_table[i].block_size)) + goto fail; + total_requested_size += cfg->block_bundles_table[i].block_size * + cfg->block_bundles_table[i].number_of_blocks; + + if (total_requested_size > new_heap->virtual_region->size || + total_requested_size % CONFIG_MM_DRV_PAGE_SIZE) + goto fail; + } + } + + /* Offset used to calculate where in virtual region to start buffer + * for the memblock + */ + uint32_t offset = 0; + + /* Iterate over all blocks and fill those that have config size value + * higher than 0. + * Each cfg entry is converted into new allocators based on mem_blocks + * API. It needs a lot of runtime allocations for members of mem_blocks. + * The allocation_sizes bit arrays are runtime allocated too. + */ + for (i = 0; i < MAX_MEMORY_ALLOCATORS_COUNT; i++) { + /* Only run for sizes > 0 */ + if (!cfg->block_bundles_table[i].block_size) + continue; + + /* bitarray_size is number of bit bundles that bittaray needs + * to represent memory blocks that's why we div round up number of blocks + * by number of bits in byte + */ + size_t bitarray_size = SOF_DIV_ROUND_UP( + SOF_DIV_ROUND_UP((uint64_t)cfg->block_bundles_table[i].number_of_blocks, 8), + sizeof(uint32_t)); + size_t bitfield_size = sizeof(uint32_t) * bitarray_size; + + /* Need to create structs that are components of mem_blocks and constantly keep + * them in memory on sys_heap. + * First create allocator - instance of sys_mem_blocks struct. + */ + struct sys_mem_blocks *new_allocator = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, + 0, SOF_MEM_CAPS_RAM, sizeof(sys_mem_blocks_t)); + + if (!new_allocator) + goto fail; + + /* Assign allocator(mem_block) to table */ + new_heap->physical_blocks_allocators[i] = new_allocator; + + /* Fill allocators data based on config and virtual region data */ + new_allocator->num_blocks = cfg->block_bundles_table[i].number_of_blocks; + new_allocator->blk_sz_shift = ilog2(cfg->block_bundles_table[i].block_size); + new_allocator->buffer = (uint8_t *)new_heap->virtual_region->addr + offset; + + /* Create bit array that is a part of mem_block kept as a ptr */ + struct sys_bitarray *allocators_bitarray = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, + 0, SOF_MEM_CAPS_RAM, sizeof(sys_bitarray_t)); + + if (!allocators_bitarray) + goto fail; + + allocators_bitarray->num_bits = cfg->block_bundles_table[i].number_of_blocks; + allocators_bitarray->num_bundles = bitarray_size; + new_allocator->bitmap = allocators_bitarray; + + /* Create bit array responsible for saving sizes of allocations + * this bit array is saved in heap struct in a table allocation_sizes + * It is also the same size as the allocator one because they will work + * together as a means to save allocation data. + * Mechanism explained in detail in free and alloc function. + */ + struct sys_bitarray *allocation_sizes_bitarray = + rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, + 0, SOF_MEM_CAPS_RAM, sizeof(sys_bitarray_t)); + + if (!allocation_sizes_bitarray) + goto fail; + + allocation_sizes_bitarray->num_bits = allocators_bitarray->num_bits; + allocation_sizes_bitarray->num_bundles = allocators_bitarray->num_bundles; + new_heap->allocation_sizes[i] = allocation_sizes_bitarray; + + /* Each sys_bitarray has a ptr to bit field that has to be allocated + * based on its size. + */ + uint32_t *allocator_bitarray_bitfield = + rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM, bitfield_size); + + if (!allocator_bitarray_bitfield) + goto fail; + + allocators_bitarray->bundles = allocator_bitarray_bitfield; + + uint32_t *sizes_bitarray_bitfield = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, + SOF_MEM_CAPS_RAM, bitfield_size); + + if (!sizes_bitarray_bitfield) + goto fail; + + new_heap->allocation_sizes[i]->bundles = sizes_bitarray_bitfield; + + /* Update offset, note that offset is CONFIG_MM_DRV_PAGE_SIZE aligned + * since every cfg size was checked against it earlier + */ + offset += cfg->block_bundles_table[i].number_of_blocks + * cfg->block_bundles_table[i].block_size; + } + + new_heap->allocating_continuously = allocating_continuously; + + /* Save the new heap pointer to a linked list */ + list_item_append(&vmh_list, &new_heap->node); + + return new_heap; +fail: + for (i = 0; i < MAX_MEMORY_ALLOCATORS_COUNT; i++) { + struct sys_mem_blocks *allocator = new_heap->physical_blocks_allocators[i]; + struct sys_bitarray *allocation_sizes = new_heap->allocation_sizes[i]; + + if (allocator) { + if (allocator->bitmap) + rfree(allocator->bitmap->bundles); + rfree(allocator->bitmap); + } + if (allocation_sizes) { + if (allocation_sizes->bundles) + rfree(allocation_sizes->bundles); + rfree(allocation_sizes); + } + rfree(allocator); + } + rfree(new_heap); + return NULL; +} + +/** + * @brief Alloc function + * + * Allocates memory on heap from provided heap pointer. + * Check if we need to map physical memory for given allocation + * and do it when necessary. + * @param heap pointer to a heap that will be used for allocation. + * @param alloc_size Size in bytes to allocate. + * + * @retval ptr to allocated memory if successful + * @retval NULL on allocation failure + */ +void *vmh_alloc(struct vmh_heap *heap, uint32_t alloc_size) +{ + if (!alloc_size) + return NULL; + /* Only operations on the same core are allowed */ + if (heap->core_id != cpu_get_id()) + return NULL; + + void *ptr = NULL; + uintptr_t phys_aligned_ptr, phys_aligned_alloc_end, phys_block_ptr; + int allocation_error_code = -ENOMEM; + size_t i, mem_block_iterator, allocation_bitarray_offset, + check_offset, check_size, check_position, block_count, + physical_block_count, block_size, allocation_bitarray_position; + uintptr_t *ptrs_to_map = NULL; + + /* We will gather error code when allocating on physical block + * allocators. + * Loop will continue as long as there are blocks to iterate on and + * if allocation keeps returning error code. + * When alloc succeeded it will return 0 on allocation and we break + * form loop and continue. + * If allocating_continuously is on for heap, allocation will be made on first available + * block series on given block bundle. For allocating_continuously off allocation will only + * happened on first free block on first allocator that can accommodate + * requested size. + */ + for (mem_block_iterator = 0; mem_block_iterator < MAX_MEMORY_ALLOCATORS_COUNT; + mem_block_iterator++) { + + /* continiue so we do not check mem blocks that do not exist */ + if (!heap->physical_blocks_allocators[mem_block_iterator]) + continue; + + /* If we do not span alloc and block is smaller than alloc we try next mem_block */ + if (block_size < alloc_size && !heap->allocating_continuously) + continue; + /* calculate block count needed to allocate for current + * mem_block. + */ + block_size = + 1 << heap->physical_blocks_allocators[mem_block_iterator]->blk_sz_shift; + block_count = SOF_DIV_ROUND_UP((uint64_t)alloc_size, (uint64_t)block_size); + + if (block_count > + heap->physical_blocks_allocators[mem_block_iterator]->num_blocks) + continue; + /* Try span alloc on first available mem_block for non span + * check if block size is sufficient. + */ + if (heap->allocating_continuously) { + allocation_error_code = sys_mem_blocks_alloc_contiguous( + heap->physical_blocks_allocators[mem_block_iterator], block_count, + &ptr); + } else if (block_size > alloc_size) { + allocation_error_code = sys_mem_blocks_alloc( + heap->physical_blocks_allocators[mem_block_iterator], block_count, + &ptr); + } + /* Save allocation data for purpose of freeing correctly + * afterwards should be changed to a implementation in mem_blocks + * that keeps alloc data there. + */ + if (allocation_error_code == 0) { + /* Mechanism of saving allocation size is a bit complicated and genius + * thought up by Adrian Warecki. + * This explanation is long and will be moved to documentation afterwards. + * Initial setup for mechanism: + * First bit array that saves allocated blocks + * in our system this will be bit array saved in mem_blocks API. + * Second bit array which is of exactly same size of first one + * that will save connection of allocation to size of allocation + * by marking if the bit of allocation is connected to next bit. + * + * Examples: lets have simple 16 bit representation of blocks + * First: 0000000000000000 allocate first four blocks 1111000000000000 + * Second:0000000000000000 1110000000000000 + * The allocator saves whether block was allocated and second bit array + * saves information by marking in position of a bit allocated 1 if + * it is a start of the allocation and has more than one + * block allocated, 1 if following bit is also part of allocation. + * 0 for individually allocated block and 0 for block ending allocation + * Continuing the example above + * First: 1111100000000000 allocate another block 1111100000000000 + * Second:1110000000000000 after the first one 1110000000000000 + * based on information above we know exactly how long allocations were + * by checking second bit array against first one + * First: 1111111100000000 allocate another block 1111111100000000 + * Second:1110000000000000 and another two blocks 1110001000000000 + * We are still able to pinpoint allocations size + */ + + /* Set bits count - 1 on the same offset as in allocation bit array */ + allocation_bitarray_offset = (uintptr_t)ptr + - (uintptr_t)heap->physical_blocks_allocators + [mem_block_iterator]->buffer; + allocation_bitarray_position = allocation_bitarray_offset / block_size; + + /* Need to set bits only if we have more than 1 block to allocate */ + if (block_count - 1) + sys_bitarray_set_region(heap->allocation_sizes[mem_block_iterator], + block_count - 1, allocation_bitarray_position); break; - case MEM_REG_ATTR_OPPORTUNISTIC_MEMORY: - vm_heaps[i].memory_caps = SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_HP; + } + } + + /* If ptr is NULL it means we did not allocate anything and we should + * break execution here + */ + if (!ptr) + return NULL; + + /* Now we need to check if have mapped physical page already + * We can do it by working out if there was any other allocation + * done in the allocator section that is in CONFIG_MM_DRV_PAGE_SIZE chunk. + * Start by getting start of allocation + * ptr aligned to physical pages boundary + */ + phys_aligned_ptr = ALIGN_DOWN((uintptr_t)ptr, CONFIG_MM_DRV_PAGE_SIZE); + phys_aligned_alloc_end = ALIGN_UP((uintptr_t)ptr + + alloc_size, CONFIG_MM_DRV_PAGE_SIZE); + + /* To check if there was any allocation done before on block we need + * to perform check operations however + * we just made allocation there. We will clear it + * from bit array of allocator for ease of operation. + * RFC if someone knows a better way to handle it in bit array API. + */ + + /* Clear bits of the allocation that was done */ + sys_bitarray_clear_region(heap->physical_blocks_allocators[mem_block_iterator]->bitmap, + block_count, allocation_bitarray_position); + + /* Now that we cleared the allocation we just made we can check + * if there was any other allocation in the space that would need to be + * mapped. Since this API is only one that uses mapping in specified address + * ranges we can assume that we made mapping for the space if it has any previous + * allocations. We check that iterating over all physical allocation blocks + */ + physical_block_count = (phys_aligned_alloc_end + - phys_aligned_ptr) / CONFIG_MM_DRV_PAGE_SIZE; + + for (i = 0; i < physical_block_count; i++) { + + phys_block_ptr = phys_aligned_ptr + i * CONFIG_MM_DRV_PAGE_SIZE; + + check_offset = phys_block_ptr + - (uintptr_t)heap->physical_blocks_allocators[mem_block_iterator]->buffer; + check_position = check_offset / block_size; + + check_size = CONFIG_MM_DRV_PAGE_SIZE / block_size; + + /* We are now sure that if the allocation bit array between + * block_realigned_ptr and block_realigned_alloc_end shows us if we already had + * any allocations there and by our logic if it needs mapping or not and + * we calculated position in bit array and size to check in bit array + */ + if (!sys_bitarray_is_region_cleared( + heap->physical_blocks_allocators[mem_block_iterator]->bitmap, + check_size, check_position)) + continue; + + /* needed in case of mid mapping failure + * Rewrite of the concept is needed. Instead of failure handling we + * need to check if there is enough physical memory available. + */ + if (!ptrs_to_map) { + ptrs_to_map = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, + 0, SOF_MEM_CAPS_RAM, sizeof(uintptr_t) * physical_block_count); + if (!ptrs_to_map) + goto fail; + } + + ptrs_to_map[i] = phys_block_ptr; + + if (sys_mm_drv_map_region((void *)phys_block_ptr, (uintptr_t)NULL, + CONFIG_MM_DRV_PAGE_SIZE, SYS_MM_MEM_PERM_RW) != 0) { + ptrs_to_map[i] = (uintptr_t)NULL; + goto fail; + } + } + + /* Set back allocation bits */ + sys_bitarray_set_region(heap->physical_blocks_allocators[mem_block_iterator]->bitmap, + block_count, allocation_bitarray_position); + rfree(ptrs_to_map); + return ptr; + +fail: + if (heap->allocating_continuously) + sys_mem_blocks_free_contiguous( + heap->physical_blocks_allocators[mem_block_iterator], ptr, block_count); + else + sys_mem_blocks_free(heap->physical_blocks_allocators[mem_block_iterator], + block_count, &ptr); + if (ptrs_to_map) { + /* If by any chance we mapped one page and couldn't map rest we need to + * free up those physical spaces. + */ + for (i = 0; i < physical_block_count; i++) { + if (ptrs_to_map[i]) + sys_mm_drv_unmap_region((void *)ptrs_to_map[i], + CONFIG_MM_DRV_PAGE_SIZE); + } + rfree(ptrs_to_map); + } + return NULL; +} + +/** + * @brief Free virtual memory heap + * + * Free the virtual memory heap object and its child allocations + * @param heap Ptr to the heap that is supposed to be freed + * + * @retval 0 on success; + * @retval -ENOTEMPTY on heap having active allocations. + */ +int vmh_free_heap(struct vmh_heap *heap) +{ + int i; + + for (i = 0; i < MAX_MEMORY_ALLOCATORS_COUNT; i++) { + if (!heap->physical_blocks_allocators[i]) + continue; + if (!sys_bitarray_is_region_cleared(heap->physical_blocks_allocators[i]->bitmap, + heap->physical_blocks_allocators[i]->num_blocks, 0)) + return -ENOTEMPTY; + } + + for (i = 0; i < MAX_MEMORY_ALLOCATORS_COUNT; i++) { + if (heap->physical_blocks_allocators[i]) { + rfree(heap->physical_blocks_allocators[i]->bitmap->bundles); + rfree(heap->physical_blocks_allocators[i]->bitmap); + rfree(heap->physical_blocks_allocators[i]); + rfree(heap->allocation_sizes[i]->bundles); + rfree(heap->allocation_sizes[i]); + } + } + list_item_del(&heap->node); + rfree(heap); + return 0; +} + +/** + * @brief Free ptr allocated on given heap + * + * Free the ptr allocation. After free action is complete + * check if any physical memory block can be freed up as + * result. + * @param heap Pointer to the heap on which ptr param is residing. + * @param ptr pointer that should be freed + * + * @retval 0 on success; + * @retval -ENOTEMPTY on heap having active allocations. + */ +int vmh_free(struct vmh_heap *heap, void *ptr) +{ + int retval; + + if (heap->core_id != cpu_get_id()) + return -EINVAL; + + size_t mem_block_iterator, i, size_to_free, block_size, ptr_bit_array_offset, + ptr_bit_array_position, physical_block_count, phys_block_end, + check_offset, check_position, check_size; + uintptr_t phys_aligned_ptr, phys_aligned_alloc_end, phys_block_ptr; + bool ptr_range_found; + + /* Get allocator from which ptr was allocated */ + for (mem_block_iterator = 0, ptr_range_found = false; + mem_block_iterator < MAX_MEMORY_ALLOCATORS_COUNT; + mem_block_iterator++) { + block_size = + 1 << heap->physical_blocks_allocators[mem_block_iterator]->blk_sz_shift; + + if (vmh_is_ptr_in_memory_range((uintptr_t)ptr, + (uintptr_t)heap->physical_blocks_allocators + [mem_block_iterator]->buffer, + heap->physical_blocks_allocators + [mem_block_iterator]->num_blocks * block_size)) { + ptr_range_found = true; break; - default: + } + } + if (!ptr_range_found) + return -EINVAL; + + /* Initially set size to block size */ + size_to_free = block_size; + + /* Get size of the allocation from ptr using allocation_sizes bit array + * and allocators bit array. Only span alloc can have different size than + * block_size. + */ + if (heap->allocating_continuously) { + + /* Not sure if that is fastest way to find the size comments welcome */ + ptr_bit_array_offset = (uintptr_t)ptr + - (uintptr_t)heap->physical_blocks_allocators[mem_block_iterator]->buffer; + ptr_bit_array_position = ptr_bit_array_offset / block_size; + + /* Allocation bit array check */ + int bit_value, prev_bit_value = 0; + + sys_bitarray_test_bit(heap->allocation_sizes[mem_block_iterator], + ptr_bit_array_position, &bit_value); + /* If checked bit is in position 0 we assume it is valid + * and assigned 0 for further logic + */ + if (ptr_bit_array_position) + sys_bitarray_test_bit(heap->allocation_sizes[mem_block_iterator], + ptr_bit_array_position - 1, &prev_bit_value); + + /* If bit is 1 we know we could be at the start of the allocation, + * we need to test previous bit from allocation if it is set then + * something went wrong in calculations. + */ + if (prev_bit_value) return -EINVAL; + + if (bit_value) { + /* We know we have more than one block was allocated so + * we need to find the size + */ + size_t bits_to_check = + heap->physical_blocks_allocators + [mem_block_iterator]->num_blocks - ptr_bit_array_position; + + /* Neeeeeeeds optimization - thinking how to do it properly + * each set bit in order after another means one allocated block. + * When we get to 0 in such range we know that is last allocated block. + * Testing bundles looks promising - need to investigate. + */ + for (i = ptr_bit_array_position; + i < bits_to_check; + i++) { + + sys_bitarray_test_bit(heap->allocation_sizes[mem_block_iterator], i, + &bit_value); + if (bit_value) + size_to_free += block_size; + else + break; + } + } else { + /* We know that there is only one block allocated + * since bit related to alloc is 0 + */ + size_to_free = block_size; } + + retval = sys_mem_blocks_free_contiguous( + heap->physical_blocks_allocators[mem_block_iterator], ptr, + size_to_free / block_size); + } else { + retval = sys_mem_blocks_free(heap->physical_blocks_allocators[mem_block_iterator], + 1, &ptr); + } + + if (retval) + return retval; + + /* Go similar route to the allocation one but + * this time marking phys pages that can be unmapped + */ + phys_aligned_ptr = ALIGN_DOWN((uintptr_t)ptr, CONFIG_MM_DRV_PAGE_SIZE); + phys_aligned_alloc_end = ALIGN_UP((uintptr_t)ptr + size_to_free, CONFIG_MM_DRV_PAGE_SIZE); + + /* Calculate how many blocks we need to check */ + physical_block_count = (phys_aligned_alloc_end + - phys_aligned_alloc_end) / CONFIG_MM_DRV_PAGE_SIZE; + + /* Unmap physical blocks that are not currently used + * we check that by looking for allocations on mem_block + * bitarrays + */ + for (i = 0; i < physical_block_count; i++) { + phys_block_ptr = phys_aligned_ptr + i * CONFIG_MM_DRV_PAGE_SIZE; + + check_offset = phys_block_ptr + - (uintptr_t)heap->physical_blocks_allocators[mem_block_iterator]->buffer; + check_position = check_offset / block_size; + + check_size = CONFIG_MM_DRV_PAGE_SIZE / block_size; + + if (sys_bitarray_is_region_cleared( + heap->physical_blocks_allocators[mem_block_iterator]->bitmap, + check_size, check_offset)) + sys_mm_drv_unmap_region((void *)phys_block_ptr, + CONFIG_MM_DRV_PAGE_SIZE); } return 0; } +/** + * @brief Reconfigure heap with given config + * + * This will destroy the heap from the pointer and recreate it + * using provided config. Individual region attribute is the + * "anchor" to the virtual space we use. + * + * @param heap Ptr to the heap that should be reconfigured + * @param cfg heap blocks configuration + * @param allocating_continuously allow contiguous on the reconfigured heap + * + * @retval heap_pointer Returns new heap pointer on success + * @retval NULL when reconfiguration failed + */ +struct vmh_heap *vmh_reconfigure_heap( + struct vmh_heap *heap, struct vmh_heap_config *cfg, + int core_id, bool allocating_continuously) +{ + uint32_t region_attribute = heap->virtual_region->attr; + + if (vmh_free_heap(heap)) + return NULL; + + return vmh_init_heap(cfg, region_attribute, core_id, allocating_continuously); +} + +/** + * @brief Get default configuration for heap + * + * This will return config created based on region provided. + * It will try to split memory in even chunks and split it into + * bundles of size 64,128,...,1024 - exactly 5 sizes. + * + * @param region Virtual region which resources we will be using for heap + * @param cfg Ptr to the struct that function will populate with data. + * + */ +void vmh_get_default_heap_config(const struct sys_mm_drv_region *region, + struct vmh_heap_config *cfg) +{ + int buffer_size = ALIGN_DOWN(region->size / DEFAULT_CONFIG_ALOCATORS_COUNT, + CONFIG_MM_DRV_PAGE_SIZE); + + int i, block_count, block_size; + + for (i = 0; i < DEFAULT_CONFIG_ALOCATORS_COUNT; i++) { + block_size = DCACHE_LINE_SIZE << i; + block_count = buffer_size / block_size; + + cfg->block_bundles_table[i].block_size = block_size; + cfg->block_bundles_table[i].number_of_blocks = block_count; + } +} + +/** + * @brief Initialization of static objects in virtual heaps API + * + * This will initialize lists of allocations and physical mappings. + * + * @param unused Variable is unused - needed by arch to clear warning. + * + * @retval Returns 0 on success + */ +static int virtual_heaps_init(void) +{ + list_init(&vmh_list); + return 0; +} + SYS_INIT(virtual_heaps_init, APPLICATION, CONFIG_APPLICATION_INIT_PRIORITY); + +/** + * @brief Gets pointer to already created heap identified by + * provided attribute. + * + * @param attr Virtual memory region attribute. + * @param core_id id of the core that heap was created for (core heap specific). + * + * @retval heap ptr on success + * @retval NULL if there was no heap created fitting the attr. + */ +struct vmh_heap *vmh_get_heap_by_attribute(uint32_t attr, uint32_t core_id) +{ + struct list_item *vm_heaps_iterator; + struct vmh_heap *retval; + + if (attr == MEM_REG_ATTR_CORE_HEAP) { + /* if we look up cpu heap we need to match its cpuid with addr + * we know that regions keep cpu heaps from 0 to max so we match + * the region of the cpu id with lists one to find match + */ + const struct sys_mm_drv_region *virtual_memory_region = + sys_mm_drv_query_memory_regions(); + /* we move ptr to cpu vmr */ + virtual_memory_region = &virtual_memory_region[core_id]; + + list_for_item(vm_heaps_iterator, &vmh_list) { + retval = + CONTAINER_OF(vm_heaps_iterator, struct vmh_heap, node); + + if (retval->virtual_region->addr == virtual_memory_region->addr) + return retval; + } + } else { + list_for_item(vm_heaps_iterator, &vmh_list) { + retval = + CONTAINER_OF(vm_heaps_iterator, struct vmh_heap, node); + if (retval->virtual_region->attr == attr) + return retval; + } + } + return NULL; +}