diff --git a/zephyr/include/sof/lib/regions_mm.h b/zephyr/include/sof/lib/regions_mm.h index 2e52cdcabf67..7e9d3a1a40a7 100644 --- a/zephyr/include/sof/lib/regions_mm.h +++ b/zephyr/include/sof/lib/regions_mm.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause */ /* - * Copyright(c) 2022 Intel Corporation. All rights reserved. + * Copyright(c) 2022 - 2023 Intel Corporation. All rights reserved. * * Author: Jakub Dabek */ @@ -10,31 +10,124 @@ #include #include +#include +#include +#include #include -#include #include -#include +#include + +/* Dependency on ipc/topology.h created due to memory capability definitions + * that are defined there + */ + +/* API is NOT re-entry safe. + * This is due to our assumption that only management code will ever handle + * memory operations on heaps themselves. + */ + +/* Defines maximum amount of memory block allocators in one heap + * since minimum realistic block should be cache line size + * and block sizes in allocator must be powers of 2 + * so logically it gives limited block sizes + * eg allocator of block size 64 128 256 512 2048 4096 1024 8192 + * would end up in 8 allocators. + * It is expected that allocations bigger than that would + * either be spanned on specifically configured heap or have + * individual configs with bigger block sizes. + */ +#define MAX_MEMORY_ALLOCATORS_COUNT 8 + +/* vmh_get_default_heap_config() function will try to split the region + * down the given count. Only applicable when API client did not + * use its config. + */ +#define DEFAULT_CONFIG_ALOCATORS_COUNT 5 + +/** @struct vmh_heap + * + * @brief This structure holds all information about virtual memory heap + * it aggregates information about its allocations and + * physical mappings. + * + * @var node generic list member used for list operations + * @var virtual_region pointer to virtual region information, it holds its + * attributes size and beginning ptr provided by zephyr. + * @var physical_blocks_allocators[] a table of block allocators + * each representing a virtual regions part in blocks of a given size + * governed by sys_mem_blocks API. + * @var allocation_sizes[] a table of bit arrays representing sizes of allocations + * made in physical_blocks_allocators directly related to physical_blocks_allocators + * @var core_id id of the core that heap was created on + * @var allocating_continuously configuration value deciding if heap allocations + * will be contiguous or single block. + */ +struct vmh_heap { + struct list_item node; + const struct sys_mm_drv_region *virtual_region; + struct sys_mem_blocks *physical_blocks_allocators[MAX_MEMORY_ALLOCATORS_COUNT]; + struct sys_bitarray *allocation_sizes[MAX_MEMORY_ALLOCATORS_COUNT]; + int core_id; + bool allocating_continuously; +}; + +/** @struct vmh_block_bundle_descriptor + * + * @brief This is a struct describing one bundle of blocks + * used as base for allocators blocks. + * + * @var block_size size of memory block. + * @var number_of_blocks number of memory blocks. + */ +struct vmh_block_bundle_descriptor { + size_t block_size; + size_t number_of_blocks; +}; /* - * Struct containing information on virtual memory heap. - * Information about allocated physical blocks is stored in - * physical_blocks_allocators variable and uses zephyr memblocks api. - */ -struct virtual_memory_heap { - /* zephyr provided virtual region */ - struct sys_mm_drv_region *virtual_region; - /* physical pages allocators represented in memblocks */ - struct sys_multi_mem_blocks physical_blocks_allocators; - /* SOF memory capability */ - uint32_t memory_caps; + * Maybe this heap config should have small bundles first going up to max + * size or there should be a sorting mechanism for those ? + * Should we assume that bundles are given from smallest to biggest ? + */ + +/** @struct vmh_heap_config + * + * @brief This is a struct that aggregates vmh_block_bundle_descriptor to + * create one cfg that can be passed to heap initiation. + * Provided config size must be physical page aligned so it + * will not overlap in physical space with other heaps during mapping. + * So every block has to have its overall size aligned to CONFIG_MM_DRV_PAGE_SIZE + * + * @vmh_block_bundle_descriptor[] aggregation of bundle descriptors. + */ +struct vmh_heap_config { + struct vmh_block_bundle_descriptor block_bundles_table[MAX_MEMORY_ALLOCATORS_COUNT]; }; -/* Available externaly array containing all information on virtual heaps - * Used to control physical allocations and overall virtual to physicall - * mapping on sof side (zephyr handles the actual assigning physical memory - * sof only requests it). +struct vmh_heap *vmh_init_heap(const struct vmh_heap_config *cfg, + int memory_region_attribute, int core_id, bool allocating_continuously); +void *vmh_alloc(struct vmh_heap *heap, uint32_t alloc_size); +int vmh_free_heap(struct vmh_heap *heap); +int vmh_free(struct vmh_heap *heap, void *ptr); +struct vmh_heap *vmh_reconfigure_heap(struct vmh_heap *heap, + struct vmh_heap_config *cfg, int core_id, bool allocating_continuously); +void vmh_get_default_heap_config(const struct sys_mm_drv_region *region, + struct vmh_heap_config *cfg); +struct vmh_heap *vmh_get_heap_by_attribute(uint32_t attr, uint32_t core_id); +/** + * @brief Checks if ptr is in range of given memory range + * + * @param ptr checked ptr + * @param range_start start of checked memory range + * @param range_size size of checked memory range. + * + * @retval false if no overlap detected. + * @retval true if any overlap detected. */ -extern struct virtual_memory_heap - vm_heaps[CONFIG_MP_MAX_NUM_CPUS + VIRTUAL_REGION_COUNT]; +static inline bool vmh_is_ptr_in_memory_range(uintptr_t ptr, uintptr_t range_start, + size_t range_size) +{ + return ptr >= range_start && ptr < range_start + range_size; +} #endif /* ZEPHYR_LIB_REGIONS_MM_H_ */ diff --git a/zephyr/lib/regions_mm.c b/zephyr/lib/regions_mm.c index 7921c3c020d3..4e3356f7fd98 100644 --- a/zephyr/lib/regions_mm.c +++ b/zephyr/lib/regions_mm.c @@ -1,52 +1,738 @@ // SPDX-License-Identifier: BSD-3-Clause /* - * Copyright(c) 2022 Intel Corporation. All rights reserved. + * Copyright(c) 2022 - 2023 Intel Corporation. All rights reserved. * * Author: Jakub Dabek */ #include -struct virtual_memory_heap - vm_heaps[CONFIG_MP_MAX_NUM_CPUS + VIRTUAL_REGION_COUNT]; +/* list of vmh_heap objects created */ +static struct list_item vmh_list; /** - * @brief Fill vm_heaps array with information from zephyr. + * @brief Initialize new heap * - * Virtual memory regions calculated in zephyr are translated here - * to a struct that will keep all information on current allocations - * and virtual to physical mappings that are related to heaps. - * System heap is not a part of this information. It only refers to - * virtual first heaps. - * Has to be initialized after calculations for regions is done in zephyr. + * Initializes new heap based on region that heap is to be assigned to + * and config. The heap size overall must be aligned to physical page + * size. + * @param cfg Configuration structure with block structure for the heap + * @param memory_region_attribute A zephyr defined virtual region identifier + * @param core_id Core id of a heap that will be created + * @param allocating_continuously Flag deciding if heap can do contiguous allocation. + * + * @retval ptr to a new heap if successful. + * @retval NULL on creation failure. */ -static int virtual_heaps_init(void) +struct vmh_heap *vmh_init_heap(const struct vmh_heap_config *cfg, + int memory_region_attribute, int core_id, bool allocating_continuously) { - struct sys_mm_drv_region *virtual_memory_regions = - (struct sys_mm_drv_region *)sys_mm_drv_query_memory_regions(); + const struct sys_mm_drv_region *virtual_memory_regions = + sys_mm_drv_query_memory_regions(); + int i; - for (size_t i = 0; - i < CONFIG_MP_MAX_NUM_CPUS + VIRTUAL_REGION_COUNT; - i++) { - vm_heaps[i].virtual_region = &virtual_memory_regions[i]; + /* Check if we haven't created heap for this region already */ + if (vmh_get_heap_by_attribute(memory_region_attribute, core_id)) + return NULL; - switch (virtual_memory_regions[i].attr) { - case MEM_REG_ATTR_CORE_HEAP: - vm_heaps[i].memory_caps = - SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_HP | SOF_MEM_CAPS_CACHE; - break; - case MEM_REG_ATTR_SHARED_HEAP: - vm_heaps[i].memory_caps = SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_HP; + struct vmh_heap *new_heap = + rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM, sizeof(*new_heap)); + + if (!new_heap) + return NULL; + + new_heap->core_id = core_id; + list_init(&new_heap->node); + struct vmh_heap_config new_config = {0}; + + /* Search for matching attribute so we place heap on right + * virtual region. + * In case of core heaps regions we count theme from 0 to max + * available cores + */ + if (memory_region_attribute == MEM_REG_ATTR_CORE_HEAP) { + new_heap->virtual_region = &virtual_memory_regions[core_id]; + } else { + for (i = CONFIG_MP_MAX_NUM_CPUS; + i < CONFIG_MP_MAX_NUM_CPUS + VIRTUAL_REGION_COUNT; i++) { + + if (memory_region_attribute == virtual_memory_regions[i].attr) { + new_heap->virtual_region = &virtual_memory_regions[i]; + break; + } + } + } + + if (!new_heap->virtual_region) + goto fail; + + /* If no config was specified we will use default one */ + if (!cfg) { + vmh_get_default_heap_config(new_heap->virtual_region, &new_config); + cfg = &new_config; + } + + /* Check if configuration provided can fit into virtual regions memory + * and if it is physical page aligned + */ + size_t total_requested_size = 0; + + for (i = 0; i < MAX_MEMORY_ALLOCATORS_COUNT; i++) { + if (cfg->block_bundles_table[i].block_size > 0) { + /* Block sizes can only be powers of 2*/ + if (!is_power_of_2(cfg->block_bundles_table[i].block_size)) + goto fail; + total_requested_size += cfg->block_bundles_table[i].block_size * + cfg->block_bundles_table[i].number_of_blocks; + + if (total_requested_size > new_heap->virtual_region->size || + total_requested_size % CONFIG_MM_DRV_PAGE_SIZE) + goto fail; + } + } + + /* Offset used to calculate where in virtual region to start buffer + * for the memblock + */ + uint32_t offset = 0; + + /* Iterate over all blocks and fill those that have config size value + * higher than 0. + * Each cfg entry is converted into new allocators based on mem_blocks + * API. It needs a lot of runtime allocations for members of mem_blocks. + * The allocation_sizes bit arrays are runtime allocated too. + */ + for (i = 0; i < MAX_MEMORY_ALLOCATORS_COUNT; i++) { + /* Only run for sizes > 0 */ + if (!cfg->block_bundles_table[i].block_size) + continue; + + /* bitarray_size is number of bit bundles that bittaray needs + * to represent memory blocks that's why we div round up number of blocks + * by number of bits in byte + */ + size_t bitarray_size = SOF_DIV_ROUND_UP( + SOF_DIV_ROUND_UP((uint64_t)cfg->block_bundles_table[i].number_of_blocks, 8), + sizeof(uint32_t)); + size_t bitfield_size = sizeof(uint32_t) * bitarray_size; + + /* Need to create structs that are components of mem_blocks and constantly keep + * them in memory on sys_heap. + * First create allocator - instance of sys_mem_blocks struct. + */ + struct sys_mem_blocks *new_allocator = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, + 0, SOF_MEM_CAPS_RAM, sizeof(sys_mem_blocks_t)); + + if (!new_allocator) + goto fail; + + /* Assign allocator(mem_block) to table */ + new_heap->physical_blocks_allocators[i] = new_allocator; + + /* Fill allocators data based on config and virtual region data */ + new_allocator->num_blocks = cfg->block_bundles_table[i].number_of_blocks; + new_allocator->blk_sz_shift = ilog2(cfg->block_bundles_table[i].block_size); + new_allocator->buffer = (uint8_t *)new_heap->virtual_region->addr + offset; + + /* Create bit array that is a part of mem_block kept as a ptr */ + struct sys_bitarray *allocators_bitarray = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, + 0, SOF_MEM_CAPS_RAM, sizeof(sys_bitarray_t)); + + if (!allocators_bitarray) + goto fail; + + allocators_bitarray->num_bits = cfg->block_bundles_table[i].number_of_blocks; + allocators_bitarray->num_bundles = bitarray_size; + new_allocator->bitmap = allocators_bitarray; + + /* Create bit array responsible for saving sizes of allocations + * this bit array is saved in heap struct in a table allocation_sizes + * It is also the same size as the allocator one because they will work + * together as a means to save allocation data. + * Mechanism explained in detail in free and alloc function. + */ + struct sys_bitarray *allocation_sizes_bitarray = + rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, + 0, SOF_MEM_CAPS_RAM, sizeof(sys_bitarray_t)); + + if (!allocation_sizes_bitarray) + goto fail; + + allocation_sizes_bitarray->num_bits = allocators_bitarray->num_bits; + allocation_sizes_bitarray->num_bundles = allocators_bitarray->num_bundles; + new_heap->allocation_sizes[i] = allocation_sizes_bitarray; + + /* Each sys_bitarray has a ptr to bit field that has to be allocated + * based on its size. + */ + uint32_t *allocator_bitarray_bitfield = + rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM, bitfield_size); + + if (!allocator_bitarray_bitfield) + goto fail; + + allocators_bitarray->bundles = allocator_bitarray_bitfield; + + uint32_t *sizes_bitarray_bitfield = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, + SOF_MEM_CAPS_RAM, bitfield_size); + + if (!sizes_bitarray_bitfield) + goto fail; + + new_heap->allocation_sizes[i]->bundles = sizes_bitarray_bitfield; + + /* Update offset, note that offset is CONFIG_MM_DRV_PAGE_SIZE aligned + * since every cfg size was checked against it earlier + */ + offset += cfg->block_bundles_table[i].number_of_blocks + * cfg->block_bundles_table[i].block_size; + } + + new_heap->allocating_continuously = allocating_continuously; + + /* Save the new heap pointer to a linked list */ + list_item_append(&vmh_list, &new_heap->node); + + return new_heap; +fail: + for (i = 0; i < MAX_MEMORY_ALLOCATORS_COUNT; i++) { + struct sys_mem_blocks *allocator = new_heap->physical_blocks_allocators[i]; + struct sys_bitarray *allocation_sizes = new_heap->allocation_sizes[i]; + + if (allocator) { + if (allocator->bitmap) + rfree(allocator->bitmap->bundles); + rfree(allocator->bitmap); + } + if (allocation_sizes) { + if (allocation_sizes->bundles) + rfree(allocation_sizes->bundles); + rfree(allocation_sizes); + } + rfree(allocator); + } + rfree(new_heap); + return NULL; +} + +/** + * @brief Alloc function + * + * Allocates memory on heap from provided heap pointer. + * Check if we need to map physical memory for given allocation + * and do it when necessary. + * @param heap pointer to a heap that will be used for allocation. + * @param alloc_size Size in bytes to allocate. + * + * @retval ptr to allocated memory if successful + * @retval NULL on allocation failure + */ +void *vmh_alloc(struct vmh_heap *heap, uint32_t alloc_size) +{ + if (!alloc_size) + return NULL; + /* Only operations on the same core are allowed */ + if (heap->core_id != cpu_get_id()) + return NULL; + + void *ptr = NULL; + uintptr_t phys_aligned_ptr, phys_aligned_alloc_end, phys_block_ptr; + int allocation_error_code = -ENOMEM; + size_t i, mem_block_iterator, allocation_bitarray_offset, + check_offset, check_size, check_position, block_count, + physical_block_count, block_size, allocation_bitarray_position; + uintptr_t *ptrs_to_map = NULL; + + /* We will gather error code when allocating on physical block + * allocators. + * Loop will continue as long as there are blocks to iterate on and + * if allocation keeps returning error code. + * When alloc succeeded it will return 0 on allocation and we break + * form loop and continue. + * If allocating_continuously is on for heap, allocation will be made on first available + * block series on given block bundle. For allocating_continuously off allocation will only + * happened on first free block on first allocator that can accommodate + * requested size. + */ + for (mem_block_iterator = 0; mem_block_iterator < MAX_MEMORY_ALLOCATORS_COUNT; + mem_block_iterator++) { + + /* continiue so we do not check mem blocks that do not exist */ + if (!heap->physical_blocks_allocators[mem_block_iterator]) + continue; + + /* If we do not span alloc and block is smaller than alloc we try next mem_block */ + if (block_size < alloc_size && !heap->allocating_continuously) + continue; + /* calculate block count needed to allocate for current + * mem_block. + */ + block_size = + 1 << heap->physical_blocks_allocators[mem_block_iterator]->blk_sz_shift; + block_count = SOF_DIV_ROUND_UP((uint64_t)alloc_size, (uint64_t)block_size); + + if (block_count > + heap->physical_blocks_allocators[mem_block_iterator]->num_blocks) + continue; + /* Try span alloc on first available mem_block for non span + * check if block size is sufficient. + */ + if (heap->allocating_continuously) { + allocation_error_code = sys_mem_blocks_alloc_contiguous( + heap->physical_blocks_allocators[mem_block_iterator], block_count, + &ptr); + } else if (block_size > alloc_size) { + allocation_error_code = sys_mem_blocks_alloc( + heap->physical_blocks_allocators[mem_block_iterator], block_count, + &ptr); + } + /* Save allocation data for purpose of freeing correctly + * afterwards should be changed to a implementation in mem_blocks + * that keeps alloc data there. + */ + if (allocation_error_code == 0) { + /* Mechanism of saving allocation size is a bit complicated and genius + * thought up by Adrian Warecki. + * This explanation is long and will be moved to documentation afterwards. + * Initial setup for mechanism: + * First bit array that saves allocated blocks + * in our system this will be bit array saved in mem_blocks API. + * Second bit array which is of exactly same size of first one + * that will save connection of allocation to size of allocation + * by marking if the bit of allocation is connected to next bit. + * + * Examples: lets have simple 16 bit representation of blocks + * First: 0000000000000000 allocate first four blocks 1111000000000000 + * Second:0000000000000000 1110000000000000 + * The allocator saves whether block was allocated and second bit array + * saves information by marking in position of a bit allocated 1 if + * it is a start of the allocation and has more than one + * block allocated, 1 if following bit is also part of allocation. + * 0 for individually allocated block and 0 for block ending allocation + * Continuing the example above + * First: 1111100000000000 allocate another block 1111100000000000 + * Second:1110000000000000 after the first one 1110000000000000 + * based on information above we know exactly how long allocations were + * by checking second bit array against first one + * First: 1111111100000000 allocate another block 1111111100000000 + * Second:1110000000000000 and another two blocks 1110001000000000 + * We are still able to pinpoint allocations size + */ + + /* Set bits count - 1 on the same offset as in allocation bit array */ + allocation_bitarray_offset = (uintptr_t)ptr + - (uintptr_t)heap->physical_blocks_allocators + [mem_block_iterator]->buffer; + allocation_bitarray_position = allocation_bitarray_offset / block_size; + + /* Need to set bits only if we have more than 1 block to allocate */ + if (block_count - 1) + sys_bitarray_set_region(heap->allocation_sizes[mem_block_iterator], + block_count - 1, allocation_bitarray_position); break; - case MEM_REG_ATTR_OPPORTUNISTIC_MEMORY: - vm_heaps[i].memory_caps = SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_HP; + } + } + + /* If ptr is NULL it means we did not allocate anything and we should + * break execution here + */ + if (!ptr) + return NULL; + + /* Now we need to check if have mapped physical page already + * We can do it by working out if there was any other allocation + * done in the allocator section that is in CONFIG_MM_DRV_PAGE_SIZE chunk. + * Start by getting start of allocation + * ptr aligned to physical pages boundary + */ + phys_aligned_ptr = ALIGN_DOWN((uintptr_t)ptr, CONFIG_MM_DRV_PAGE_SIZE); + phys_aligned_alloc_end = ALIGN_UP((uintptr_t)ptr + + alloc_size, CONFIG_MM_DRV_PAGE_SIZE); + + /* To check if there was any allocation done before on block we need + * to perform check operations however + * we just made allocation there. We will clear it + * from bit array of allocator for ease of operation. + * RFC if someone knows a better way to handle it in bit array API. + */ + + /* Clear bits of the allocation that was done */ + sys_bitarray_clear_region(heap->physical_blocks_allocators[mem_block_iterator]->bitmap, + block_count, allocation_bitarray_position); + + /* Now that we cleared the allocation we just made we can check + * if there was any other allocation in the space that would need to be + * mapped. Since this API is only one that uses mapping in specified address + * ranges we can assume that we made mapping for the space if it has any previous + * allocations. We check that iterating over all physical allocation blocks + */ + physical_block_count = (phys_aligned_alloc_end + - phys_aligned_ptr) / CONFIG_MM_DRV_PAGE_SIZE; + + for (i = 0; i < physical_block_count; i++) { + + phys_block_ptr = phys_aligned_ptr + i * CONFIG_MM_DRV_PAGE_SIZE; + + check_offset = phys_block_ptr + - (uintptr_t)heap->physical_blocks_allocators[mem_block_iterator]->buffer; + check_position = check_offset / block_size; + + check_size = CONFIG_MM_DRV_PAGE_SIZE / block_size; + + /* We are now sure that if the allocation bit array between + * block_realigned_ptr and block_realigned_alloc_end shows us if we already had + * any allocations there and by our logic if it needs mapping or not and + * we calculated position in bit array and size to check in bit array + */ + if (!sys_bitarray_is_region_cleared( + heap->physical_blocks_allocators[mem_block_iterator]->bitmap, + check_size, check_position)) + continue; + + /* needed in case of mid mapping failure + * Rewrite of the concept is needed. Instead of failure handling we + * need to check if there is enough physical memory available. + */ + if (!ptrs_to_map) { + ptrs_to_map = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, + 0, SOF_MEM_CAPS_RAM, sizeof(uintptr_t) * physical_block_count); + if (!ptrs_to_map) + goto fail; + } + + ptrs_to_map[i] = phys_block_ptr; + + if (sys_mm_drv_map_region((void *)phys_block_ptr, (uintptr_t)NULL, + CONFIG_MM_DRV_PAGE_SIZE, SYS_MM_MEM_PERM_RW) != 0) { + ptrs_to_map[i] = (uintptr_t)NULL; + goto fail; + } + } + + /* Set back allocation bits */ + sys_bitarray_set_region(heap->physical_blocks_allocators[mem_block_iterator]->bitmap, + block_count, allocation_bitarray_position); + rfree(ptrs_to_map); + return ptr; + +fail: + if (heap->allocating_continuously) + sys_mem_blocks_free_contiguous( + heap->physical_blocks_allocators[mem_block_iterator], ptr, block_count); + else + sys_mem_blocks_free(heap->physical_blocks_allocators[mem_block_iterator], + block_count, &ptr); + if (ptrs_to_map) { + /* If by any chance we mapped one page and couldn't map rest we need to + * free up those physical spaces. + */ + for (i = 0; i < physical_block_count; i++) { + if (ptrs_to_map[i]) + sys_mm_drv_unmap_region((void *)ptrs_to_map[i], + CONFIG_MM_DRV_PAGE_SIZE); + } + rfree(ptrs_to_map); + } + return NULL; +} + +/** + * @brief Free virtual memory heap + * + * Free the virtual memory heap object and its child allocations + * @param heap Ptr to the heap that is supposed to be freed + * + * @retval 0 on success; + * @retval -ENOTEMPTY on heap having active allocations. + */ +int vmh_free_heap(struct vmh_heap *heap) +{ + int i; + + for (i = 0; i < MAX_MEMORY_ALLOCATORS_COUNT; i++) { + if (!heap->physical_blocks_allocators[i]) + continue; + if (!sys_bitarray_is_region_cleared(heap->physical_blocks_allocators[i]->bitmap, + heap->physical_blocks_allocators[i]->num_blocks, 0)) + return -ENOTEMPTY; + } + + for (i = 0; i < MAX_MEMORY_ALLOCATORS_COUNT; i++) { + if (heap->physical_blocks_allocators[i]) { + rfree(heap->physical_blocks_allocators[i]->bitmap->bundles); + rfree(heap->physical_blocks_allocators[i]->bitmap); + rfree(heap->physical_blocks_allocators[i]); + rfree(heap->allocation_sizes[i]->bundles); + rfree(heap->allocation_sizes[i]); + } + } + list_item_del(&heap->node); + rfree(heap); + return 0; +} + +/** + * @brief Free ptr allocated on given heap + * + * Free the ptr allocation. After free action is complete + * check if any physical memory block can be freed up as + * result. + * @param heap Pointer to the heap on which ptr param is residing. + * @param ptr pointer that should be freed + * + * @retval 0 on success; + * @retval -ENOTEMPTY on heap having active allocations. + */ +int vmh_free(struct vmh_heap *heap, void *ptr) +{ + int retval; + + if (heap->core_id != cpu_get_id()) + return -EINVAL; + + size_t mem_block_iterator, i, size_to_free, block_size, ptr_bit_array_offset, + ptr_bit_array_position, physical_block_count, phys_block_end, + check_offset, check_position, check_size; + uintptr_t phys_aligned_ptr, phys_aligned_alloc_end, phys_block_ptr; + bool ptr_range_found; + + /* Get allocator from which ptr was allocated */ + for (mem_block_iterator = 0, ptr_range_found = false; + mem_block_iterator < MAX_MEMORY_ALLOCATORS_COUNT; + mem_block_iterator++) { + block_size = + 1 << heap->physical_blocks_allocators[mem_block_iterator]->blk_sz_shift; + + if (vmh_is_ptr_in_memory_range((uintptr_t)ptr, + (uintptr_t)heap->physical_blocks_allocators + [mem_block_iterator]->buffer, + heap->physical_blocks_allocators + [mem_block_iterator]->num_blocks * block_size)) { + ptr_range_found = true; break; - default: + } + } + if (!ptr_range_found) + return -EINVAL; + + /* Initially set size to block size */ + size_to_free = block_size; + + /* Get size of the allocation from ptr using allocation_sizes bit array + * and allocators bit array. Only span alloc can have different size than + * block_size. + */ + if (heap->allocating_continuously) { + + /* Not sure if that is fastest way to find the size comments welcome */ + ptr_bit_array_offset = (uintptr_t)ptr + - (uintptr_t)heap->physical_blocks_allocators[mem_block_iterator]->buffer; + ptr_bit_array_position = ptr_bit_array_offset / block_size; + + /* Allocation bit array check */ + int bit_value, prev_bit_value = 0; + + sys_bitarray_test_bit(heap->allocation_sizes[mem_block_iterator], + ptr_bit_array_position, &bit_value); + /* If checked bit is in position 0 we assume it is valid + * and assigned 0 for further logic + */ + if (ptr_bit_array_position) + sys_bitarray_test_bit(heap->allocation_sizes[mem_block_iterator], + ptr_bit_array_position - 1, &prev_bit_value); + + /* If bit is 1 we know we could be at the start of the allocation, + * we need to test previous bit from allocation if it is set then + * something went wrong in calculations. + */ + if (prev_bit_value) return -EINVAL; + + if (bit_value) { + /* We know we have more than one block was allocated so + * we need to find the size + */ + size_t bits_to_check = + heap->physical_blocks_allocators + [mem_block_iterator]->num_blocks - ptr_bit_array_position; + + /* Neeeeeeeds optimization - thinking how to do it properly + * each set bit in order after another means one allocated block. + * When we get to 0 in such range we know that is last allocated block. + * Testing bundles looks promising - need to investigate. + */ + for (i = ptr_bit_array_position; + i < bits_to_check; + i++) { + + sys_bitarray_test_bit(heap->allocation_sizes[mem_block_iterator], i, + &bit_value); + if (bit_value) + size_to_free += block_size; + else + break; + } + } else { + /* We know that there is only one block allocated + * since bit related to alloc is 0 + */ + size_to_free = block_size; } + + retval = sys_mem_blocks_free_contiguous( + heap->physical_blocks_allocators[mem_block_iterator], ptr, + size_to_free / block_size); + } else { + retval = sys_mem_blocks_free(heap->physical_blocks_allocators[mem_block_iterator], + 1, &ptr); + } + + if (retval) + return retval; + + /* Go similar route to the allocation one but + * this time marking phys pages that can be unmapped + */ + phys_aligned_ptr = ALIGN_DOWN((uintptr_t)ptr, CONFIG_MM_DRV_PAGE_SIZE); + phys_aligned_alloc_end = ALIGN_UP((uintptr_t)ptr + size_to_free, CONFIG_MM_DRV_PAGE_SIZE); + + /* Calculate how many blocks we need to check */ + physical_block_count = (phys_aligned_alloc_end + - phys_aligned_alloc_end) / CONFIG_MM_DRV_PAGE_SIZE; + + /* Unmap physical blocks that are not currently used + * we check that by looking for allocations on mem_block + * bitarrays + */ + for (i = 0; i < physical_block_count; i++) { + phys_block_ptr = phys_aligned_ptr + i * CONFIG_MM_DRV_PAGE_SIZE; + + check_offset = phys_block_ptr + - (uintptr_t)heap->physical_blocks_allocators[mem_block_iterator]->buffer; + check_position = check_offset / block_size; + + check_size = CONFIG_MM_DRV_PAGE_SIZE / block_size; + + if (sys_bitarray_is_region_cleared( + heap->physical_blocks_allocators[mem_block_iterator]->bitmap, + check_size, check_offset)) + sys_mm_drv_unmap_region((void *)phys_block_ptr, + CONFIG_MM_DRV_PAGE_SIZE); } return 0; } +/** + * @brief Reconfigure heap with given config + * + * This will destroy the heap from the pointer and recreate it + * using provided config. Individual region attribute is the + * "anchor" to the virtual space we use. + * + * @param heap Ptr to the heap that should be reconfigured + * @param cfg heap blocks configuration + * @param allocating_continuously allow contiguous on the reconfigured heap + * + * @retval heap_pointer Returns new heap pointer on success + * @retval NULL when reconfiguration failed + */ +struct vmh_heap *vmh_reconfigure_heap( + struct vmh_heap *heap, struct vmh_heap_config *cfg, + int core_id, bool allocating_continuously) +{ + uint32_t region_attribute = heap->virtual_region->attr; + + if (vmh_free_heap(heap)) + return NULL; + + return vmh_init_heap(cfg, region_attribute, core_id, allocating_continuously); +} + +/** + * @brief Get default configuration for heap + * + * This will return config created based on region provided. + * It will try to split memory in even chunks and split it into + * bundles of size 64,128,...,1024 - exactly 5 sizes. + * + * @param region Virtual region which resources we will be using for heap + * @param cfg Ptr to the struct that function will populate with data. + * + */ +void vmh_get_default_heap_config(const struct sys_mm_drv_region *region, + struct vmh_heap_config *cfg) +{ + int buffer_size = ALIGN_DOWN(region->size / DEFAULT_CONFIG_ALOCATORS_COUNT, + CONFIG_MM_DRV_PAGE_SIZE); + + int i, block_count, block_size; + + for (i = 0; i < DEFAULT_CONFIG_ALOCATORS_COUNT; i++) { + block_size = DCACHE_LINE_SIZE << i; + block_count = buffer_size / block_size; + + cfg->block_bundles_table[i].block_size = block_size; + cfg->block_bundles_table[i].number_of_blocks = block_count; + } +} + +/** + * @brief Initialization of static objects in virtual heaps API + * + * This will initialize lists of allocations and physical mappings. + * + * @param unused Variable is unused - needed by arch to clear warning. + * + * @retval Returns 0 on success + */ +static int virtual_heaps_init(void) +{ + list_init(&vmh_list); + return 0; +} + SYS_INIT(virtual_heaps_init, APPLICATION, CONFIG_APPLICATION_INIT_PRIORITY); + +/** + * @brief Gets pointer to already created heap identified by + * provided attribute. + * + * @param attr Virtual memory region attribute. + * @param core_id id of the core that heap was created for (core heap specific). + * + * @retval heap ptr on success + * @retval NULL if there was no heap created fitting the attr. + */ +struct vmh_heap *vmh_get_heap_by_attribute(uint32_t attr, uint32_t core_id) +{ + struct list_item *vm_heaps_iterator; + struct vmh_heap *retval; + + if (attr == MEM_REG_ATTR_CORE_HEAP) { + /* if we look up cpu heap we need to match its cpuid with addr + * we know that regions keep cpu heaps from 0 to max so we match + * the region of the cpu id with lists one to find match + */ + const struct sys_mm_drv_region *virtual_memory_region = + sys_mm_drv_query_memory_regions(); + /* we move ptr to cpu vmr */ + virtual_memory_region = &virtual_memory_region[core_id]; + + list_for_item(vm_heaps_iterator, &vmh_list) { + retval = + CONTAINER_OF(vm_heaps_iterator, struct vmh_heap, node); + + if (retval->virtual_region->addr == virtual_memory_region->addr) + return retval; + } + } else { + list_for_item(vm_heaps_iterator, &vmh_list) { + retval = + CONTAINER_OF(vm_heaps_iterator, struct vmh_heap, node); + if (retval->virtual_region->attr == attr) + return retval; + } + } + return NULL; +}