diff --git a/.gitmodules b/.gitmodules index 9fad6e8e..53218b17 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "shared"] path = shared - url = https://github.com/differrari/redlib + url = https://github.com/differrari/redlib.git diff --git a/bin/read/linker.ld b/bin/read/linker.ld index 86f59536..e940a7ff 100644 --- a/bin/read/linker.ld +++ b/bin/read/linker.ld @@ -1,25 +1,30 @@ ENTRY(main) +PHDRS { + text PT_LOAD FLAGS(5); + data PT_LOAD FLAGS(6); +} SECTIONS { . = 0x20000; .text : { *(.text .text.*) - } + }:text .rodata : { *(.rodata .rodata.*) - } + } :text + . = ALIGN(0x1000); .data : { *(.data .data.*) - } + } :data - .bss : { + .bss (NOLOAD) : { __bss_start = .; *(.bss .bss.* COMMON) __bss_end = .; - } + } :data /DISCARD/ : { *(.comment .note .eh_frame) diff --git a/bin/test/linker.ld b/bin/test/linker.ld index 86f59536..e940a7ff 100644 --- a/bin/test/linker.ld +++ b/bin/test/linker.ld @@ -1,25 +1,30 @@ ENTRY(main) +PHDRS { + text PT_LOAD FLAGS(5); + data PT_LOAD FLAGS(6); +} SECTIONS { . = 0x20000; .text : { *(.text .text.*) - } + }:text .rodata : { *(.rodata .rodata.*) - } + } :text + . = ALIGN(0x1000); .data : { *(.data .data.*) - } + } :data - .bss : { + .bss (NOLOAD) : { __bss_start = .; *(.bss .bss.* COMMON) __bss_end = .; - } + } :data /DISCARD/ : { *(.comment .note .eh_frame) diff --git a/common.mk b/common.mk index 4e6aaace..30b12b37 100644 --- a/common.mk +++ b/common.mk @@ -14,7 +14,7 @@ BUILD_DIR := ./build COMMON_FLAGS ?= -ffreestanding -nostdlib -fno-exceptions -fno-unwind-tables \ -fno-asynchronous-unwind-tables -g -O0 -Wall -Wextra \ -Wno-unused-parameter -Wno-address-of-packed-member \ - -Werror \ + #-Werror \ -Wno-unused-function ifeq ($(ARCH), aarch64-none-elf-) diff --git a/kernel/audio/OutputAudioDevice.cpp b/kernel/audio/OutputAudioDevice.cpp index 89084875..401917a1 100644 --- a/kernel/audio/OutputAudioDevice.cpp +++ b/kernel/audio/OutputAudioDevice.cpp @@ -8,6 +8,7 @@ void OutputAudioDevice::populate(){ } sizedptr OutputAudioDevice::request_buffer(){ + *(uint32_t*)(buffer + write_ptr) = stream_id; sizedptr ptr = (sizedptr){buffer + write_ptr + header_size, buf_size}; write_ptr += packet_size; if (write_ptr + packet_size >= BUF_SIZE) write_ptr = 0; diff --git a/kernel/bin/bin_mod.c b/kernel/bin/bin_mod.c index 23b06a0d..8bf1eeef 100644 --- a/kernel/bin/bin_mod.c +++ b/kernel/bin/bin_mod.c @@ -9,8 +9,10 @@ #include "console/kio.h" #include "process/loading/elf_file.h" #include "memory/page_allocator.h" +#include "alloc/allocate.h" #include "std/memory.h" #include "sysregs.h" +#include "memory/addr.h" bool init_bin(){ return true; @@ -29,64 +31,94 @@ open_bin_ref available_cmds[] = { }; process_t* execute(const char* prog_name, int argc, const char* argv[]){ - size_t listsize = 0x1000; - void *listptr = malloc(listsize); - if (list_directory_contents("/boot/redos/bin/", listptr, listsize, 0)){ - size_t name_len = strlen(prog_name) + 4; - char *full_name = strcat_new(prog_name, ".elf"); - string_list *list = (string_list*)listptr; - char* reader = (char*)list->array; - kprintf("Directory contains %i files",list->count); - for (uint32_t i = 0; i < list->count; i++){ - char *f = reader; - if (*f){ - if (strcmp_case(f, full_name,true) == 0){ - string path = string_format("/boot/redos/bin/%s",full_name); - file fd = {}; - FS_RESULT op = openf(path.data, &fd); - if (op != FS_RESULT_SUCCESS){ - kprintf("Failed to open file %s",path.data); - return 0; + if (!prog_name || !*prog_name) return 0; + + char *full_name = 0; + full_name = (strend_case(prog_name, ".elf", true) == 0) ? string_from_literal(prog_name).data : strcat_new(prog_name, ".elf"); + if (full_name) { + char pathbuf[1024]; + size_t pathlen = string_format_buf(pathbuf, sizeof(pathbuf), "/boot/redos/bin/%s",full_name); + if (pathlen >= sizeof(pathbuf) - 1) { + release(full_name); + return 0; + } + + file fd = {}; + FS_RESULT op = openf(pathbuf, &fd); + if (op == FS_RESULT_SUCCESS){ + char *program = zalloc(fd.size); + if (!program){ + kprintf("Failed to read file %s", pathbuf); + closef(&fd); + release(full_name); + return 0; + } + if (readf(&fd, program, fd.size) != fd.size){ + kprintf("Failed to read file %s", pathbuf); + closef(&fd); + release(program); + release(full_name); + return 0; + } + process_t *proc = load_elf_file(prog_name, 0, program, fd.size); + closef(&fd); + release(program); + if (!proc){ + kprintf("Failed to create process for %s",prog_name); + release(full_name); + return 0; + } + proc->PROC_X0 = argc; + proc->PROC_X1 = 0; + if (argc > 0 && argv) { + bool args_ok = true; + size_t total_str = 0; + for (int j = 0; j < argc; j++) { + const char *s = argv[j]; + if (!s) { + args_ok = false; + break; } - char *program = malloc(fd.size); - if (readf(&fd, program, fd.size) != fd.size){ - kprintf("Failed to read file %s", path.data); + size_t l = strlen(s); + if (l > proc->stack_size) { + args_ok = false; + break; } - process_t *proc = load_elf_file(prog_name, 0, program, fd.size); - string_free(path); - free_sized(full_name, name_len); - closef(&fd); - if (!proc){ - kprintf("Failed to create process for %s",prog_name); + if (total_str > proc->stack_size - (l + 1)) { + args_ok = false; + break; } - proc->PROC_X0 = argc; - if (argc > 0){ - uintptr_t start = (uintptr_t)argv[0]; - uintptr_t end = (uintptr_t)argv; - size_t total = end-start; - size_t argvs = argc * sizeof(uintptr_t); - char *nargvals = (char*)(PHYS_TO_VIRT_P(proc->stack_phys)-total-argvs); - char *vnargvals = (char*)(proc->stack-total-argvs); - char** nargv = (char**)(PHYS_TO_VIRT_P(proc->stack_phys)-argvs); - uintptr_t strptr = 0; - for (int i = 0; i < argc; i++){ - size_t strsize = strlen(argv[i]); - memcpy(nargvals + strptr, argv[i], strsize); - *(char*)(nargvals + strptr + strsize++) = 0; - nargv[i] = vnargvals + strptr; - strptr += strsize; - } - proc->PROC_X1 = (uintptr_t)proc->stack-argvs; - proc->sp -= total+argvs; + total_str += l + 1; + } + + size_t argvs = (size_t)argc * sizeof(uintptr_t); + if (argvs > proc->stack_size) args_ok = false; + if (total_str> proc->stack_size - argvs) args_ok = false; + size_t total = total_str + argvs; + if (total + 16 > proc->stack_size) args_ok = false; + if (args_ok) { + char *nargvals = (char*)((void*)dmap_pa_to_kva(proc->stack_phys)-total); + char *vnargvals = (char*)(proc->stack-total); + char** nargv = (char**)((void*)dmap_pa_to_kva(proc->stack_phys)-argvs); + size_t strptr = 0; + for (int j = 0; j < argc; j++){ + size_t strsize = strlen(argv[j]); + memcpy(nargvals + strptr, argv[j], strsize); + nargvals[strptr + strsize] = 0; + nargv[j] = vnargvals + strptr; + strptr += strsize+1; } - proc->state = READY; - return proc; + proc->PROC_X1 = (uintptr_t)proc->stack-argvs; + proc->sp = (proc->stack - total) & ~0xFULL; + } else { + proc->PROC_X0 = 0; } - while (*reader) reader++; - reader++; } + proc->state = READY; + release(full_name); + return proc; } - free_sized(full_name,name_len); + release(full_name); } for (uint32_t i = 0; i < N_ARR(available_cmds); i++){ diff --git a/kernel/bin/monitor_processes.c b/kernel/bin/monitor_processes.c index e18a89ba..b56cd339 100644 --- a/kernel/bin/monitor_processes.c +++ b/kernel/bin/monitor_processes.c @@ -73,7 +73,7 @@ void draw_memory(char *name,int x, int y, int width, int full_height, int used, string str = string_format("%s\n%x",(uintptr_t)name, used); fb_draw_string(&ctx,str.data, stack_top.x, stack_top.y + height + 5, 2, system_theme.bg_color); - free_sized(str.data,str.mem_length); + string_free(str); } void draw_process_view(){ @@ -130,7 +130,7 @@ void draw_process_view(){ string pc = string_from_hex(proc->pc); fb_draw_string(&ctx,pc.data, xo, pc_y, scale, system_theme.bg_color); - free_sized(pc.data, pc.mem_length); + string_free(pc); draw_memory("Stack", xo, stack_y, stack_width, stack_height, proc->stack - proc->sp, proc->stack_size); uint64_t heap = calc_heap(proc->heap_phys); @@ -139,9 +139,9 @@ void draw_process_view(){ string flags = string_format("Flags: %x", proc->spsr); fb_draw_string(&ctx, flags.data, xo, flags_y, scale, system_theme.bg_color); - free_sized(name.data, name.mem_length); - free_sized(state.data, state.mem_length); - free_sized(flags.data, flags.mem_length); + string_free(name); + string_free(state); + string_free(flags); } commit_draw_ctx(&ctx); diff --git a/kernel/bin/ping.c b/kernel/bin/ping.c index 50bf6e48..b8c57f81 100644 --- a/kernel/bin/ping.c +++ b/kernel/bin/ping.c @@ -106,7 +106,7 @@ static int ping_v4(file *fd, const ping_opts_t *o) { string m = string_format("ping: dns lookup failed (%d) for '%s'", (int)dr, host); write_file(fd, m.data, m.length); write_file(fd, "\n", 1); - free_sized(m.data, m.mem_length); + string_free(m); return 2; } dst_ip_be = r; @@ -156,7 +156,7 @@ static int ping_v4(file *fd, const ping_opts_t *o) { string ln = string_format("Reply from %s: bytes=32 time=%ums", ipstr, (uint32_t)rtt); write_file(fd, ln.data, ln.length); write_file(fd, "\n", 1); - free_sized(ln.data, ln.mem_length); + string_free(ln); } else { const char *msg = status_to_msg(res.status); write_file(fd, msg, strlen(msg)); @@ -171,7 +171,7 @@ static int ping_v4(file *fd, const ping_opts_t *o) { string h = string_format("--- %s ping statistics ---", host); write_file(fd, h.data, h.length); write_file(fd, "\n", 1); - free_sized(h.data, h.mem_length); + string_free(h); uint32_t loss = (sent == 0) ? 0 : (uint32_t)((((uint64_t)(sent - received)) * 100) / sent); uint32_t total_time = (o->count > 0) ? (o->count - 1) * o->interval_ms : 0; @@ -179,7 +179,7 @@ static int ping_v4(file *fd, const ping_opts_t *o) { string s = string_format("%u packets transmitted, %u received, %u%% packet loss, time %ums", sent, received, loss, total_time); write_file(fd, s.data, s.length); write_file(fd, "\n", 1); - free_sized(s.data, s.mem_length); + string_free(s); if (received > 0) { uint32_t avg = (uint32_t)(sum_ms / received); @@ -187,7 +187,7 @@ static int ping_v4(file *fd, const ping_opts_t *o) { string r = string_format("rtt min/avg/max = %u/%u/%u ms", min_ms, avg, max_ms); write_file(fd, r.data, r.length); write_file(fd, "\n", 1); - free_sized(r.data, r.mem_length); + string_free(r); } return (received > 0) ? 0 : 1; @@ -204,7 +204,7 @@ static int ping_v6(file *fd, const ping_opts_t *o) { string m = string_format("ping: dns lookup failed (%d) for '%s'",(int)dr, host); write_file(fd, m.data, m.length); write_file(fd, "\n", 1); - free_sized(m.data, m.mem_length); + string_free(m); return 2; } } @@ -241,7 +241,7 @@ static int ping_v6(file *fd, const ping_opts_t *o) { string ln = string_format("Reply from %s: bytes=32 time=%ums", ipstr, (uint32_t)rtt); write_file(fd, ln.data, ln.length); write_file(fd, "\n", 1); - free_sized(ln.data, ln.mem_length); + string_free(ln); } else { const char *msg = status_to_msg(res.status); write_file(fd, msg, strlen(msg)); @@ -256,7 +256,7 @@ static int ping_v6(file *fd, const ping_opts_t *o) { string h = string_format("--- %s ping statistics ---", host); write_file(fd, h.data, h.length); write_file(fd, "\n", 1); - free_sized(h.data, h.mem_length); + string_free(h); uint32_t loss = (sent == 0) ? 0 : (uint32_t)((((uint64_t)(sent - received)) * 100) / sent); @@ -265,7 +265,7 @@ static int ping_v6(file *fd, const ping_opts_t *o) { string s = string_format("%u packets transmitted, %u received, %u%% packet loss, time %ums", sent, received, loss, total_time); write_file(fd, s.data, s.length); write_file(fd, "\n", 1); - free_sized(s.data, s.mem_length); + string_free(s); if (received > 0) { uint32_t avg = (uint32_t)(sum_ms / received); @@ -273,7 +273,7 @@ static int ping_v6(file *fd, const ping_opts_t *o) { string r = string_format("rtt min/avg/max = %u/%u/%u ms", min_ms, avg, max_ms); write_file(fd, r.data, r.length); write_file(fd, "\n", 1); - free_sized(r.data, r.mem_length); + string_free(r); } return (received > 0) ? 0 : 1; @@ -284,7 +284,7 @@ int run_ping(int argc, char *argv[]) { string p = string_format("/proc/%u/out", pid); file fd = {0}; open_file(p.data, &fd); - free_sized(p.data, p.mem_length); + string_free(p); ping_opts_t opts; if (!parse_args(argc, argv, &opts)) { @@ -308,7 +308,7 @@ int run_ping(int argc, char *argv[]) { string em = string_format("ping: invalid source %s (no local ip match)", ssrc); write_file(&fd, em.data, em.length); write_file(&fd, "\n", 1); - free_sized(em.data, em.mem_length); + string_free(em); close_file(&fd); return 2; } diff --git a/kernel/bin/shutdown.c b/kernel/bin/shutdown.c index f74abb15..4d987734 100644 --- a/kernel/bin/shutdown.c +++ b/kernel/bin/shutdown.c @@ -15,7 +15,7 @@ int run_shutdown(int argc, char* argv[]){ string p = string_format("/proc/%i/out", pid); file out; FS_RESULT r = open_file(p.data, &out); - free_sized(p.data, p.mem_length); + string_free(p); if (r != FS_RESULT_SUCCESS){ return 2; diff --git a/kernel/bin/tracert.c b/kernel/bin/tracert.c index 98da6a32..50b5fd8e 100644 --- a/kernel/bin/tracert.c +++ b/kernel/bin/tracert.c @@ -101,7 +101,7 @@ static int tracert_v4(file *fd, const tr_opts_t *o) { write_file(fd, m.data, m.length); write_file(fd, "\n", 1); //write_file(fd, "\t", 1); - free_sized(m.data, m.mem_length); + string_free(m); return 2; } dst = r; @@ -120,7 +120,7 @@ static int tracert_v4(file *fd, const tr_opts_t *o) { for (uint32_t p = 0; p < o->count; p++) { string col = string_format("rtt%u ", (uint32_t)(p + 1)); write_file(fd, col.data, col.length); - free_sized(col.data, col.mem_length); + string_free(col); } write_file(fd, "address", 7); write_file(fd, "\n", 1); @@ -135,7 +135,7 @@ static int tracert_v4(file *fd, const tr_opts_t *o) { string em = string_format("tracert: invalid source %s (no local ip match)", ssrc); write_file(fd, em.data, em.length); write_file(fd, "\n", 1); - free_sized(em.data, em.mem_length); + string_free(em); return 2; } txo.index = l3->l3_id; @@ -150,7 +150,7 @@ static int tracert_v4(file *fd, const tr_opts_t *o) { for (uint32_t ttl = 1; ttl <= o->max_ttl; ttl++) { string hdr = string_format("%2u ", ttl); write_file(fd, hdr.data, hdr.length); - free_sized(hdr.data, hdr.mem_length); + string_free(hdr); uint32_t hop_ip = 0; bool any = false; @@ -165,7 +165,7 @@ static int tracert_v4(file *fd, const tr_opts_t *o) { any = true; string ms = string_format("%ums ", r.rtt_ms); write_file(fd, ms.data, ms.length); - free_sized(ms.data, ms.mem_length); + string_free(ms); } else { if (r.status == PING_TTL_EXPIRED || r.status == PING_REDIRECT || r.status == PING_PARAM_PROBLEM || r.status == PING_NET_UNREACH || r.status == PING_HOST_UNREACH || r.status == PING_ADMIN_PROHIBITED || @@ -173,7 +173,7 @@ static int tracert_v4(file *fd, const tr_opts_t *o) { any = true; string ms = string_format("%ums ", r.rtt_ms); write_file(fd, ms.data, ms.length); - free_sized(ms.data, ms.mem_length); + string_free(ms); } else { write_file(fd, "* ", 3); } @@ -202,7 +202,7 @@ static int tracert_v4(file *fd, const tr_opts_t *o) { string note = string_format("stopping after %u consecutive timeout hops", dead_streak); write_file(fd, note.data, note.length); write_file(fd, "\n", 1); - free_sized(note.data, note.mem_length); + string_free(note); break; } } @@ -219,7 +219,7 @@ static int tracert_v6(file *fd, const tr_opts_t *o) { string m = string_format("tracert: dns lookup failed (%d) for '%s'", (int)dr, o->host); write_file(fd, m.data, m.length); write_file(fd, "\n", 1); - free_sized(m.data, m.mem_length); + string_free(m); return 2; } } @@ -236,7 +236,7 @@ static int tracert_v6(file *fd, const tr_opts_t *o) { for (uint32_t p = 0; p < o->count; p++) { string col = string_format("rtt%u ", p + 1); write_file(fd, col.data, col.length); - free_sized(col.data, col.mem_length); + string_free(col); } write_file(fd, "address\n", 8); @@ -247,7 +247,7 @@ static int tracert_v6(file *fd, const tr_opts_t *o) { for (uint32_t hl = 1; hl <= o->max_ttl; hl++) { string h = string_format("%2u ", hl); write_file(fd, h.data, h.length); - free_sized(h.data, h.mem_length); + string_free(h); uint8_t hop_ip[16]; for (int i = 0; i < 16; i++) hop_ip[i] = 0; @@ -264,7 +264,7 @@ static int tracert_v6(file *fd, const tr_opts_t *o) { any = true; string ms = string_format("%ums ", r.rtt_ms); write_file(fd, ms.data, ms.length); - free_sized(ms.data, ms.mem_length); + string_free(ms); } else { if (r.status == PING_TTL_EXPIRED || r.status == PING_REDIRECT || r.status == PING_PARAM_PROBLEM || r.status == PING_NET_UNREACH || @@ -273,7 +273,7 @@ static int tracert_v6(file *fd, const tr_opts_t *o) { any = true; string ms = string_format("%ums ", r.rtt_ms); write_file(fd, ms.data, ms.length); - free_sized(ms.data, ms.mem_length); + string_free(ms); } else { write_file(fd, "* ", 3); } @@ -302,7 +302,7 @@ static int tracert_v6(file *fd, const tr_opts_t *o) { string note = string_format("stopping after %u consecutive timeout hops", dead_streak); write_file(fd, note.data, note.length); write_file(fd, "\n", 1); - free_sized(note.data, note.mem_length); + string_free(note); break; } } @@ -315,7 +315,7 @@ int run_tracert(int argc, char *argv[]) { string p = string_format("/proc/%u/out", pid); file fd = (file){0}; open_file(p.data, &fd); - free_sized(p.data, p.mem_length); + string_free(p); tr_opts_t o; if (!parse_args(argc, argv, &o)) { diff --git a/kernel/boot.S b/kernel/boot.S index 71588b37..fc646599 100644 --- a/kernel/boot.S +++ b/kernel/boot.S @@ -1,8 +1,50 @@ #include "sysregs.h" +.section .boot.data,"aw" +.align 3 + +.global boot_args //uint64_t boot_args[3] +boot_args: + .zero 3*8 + +.global boot_fault_info_el1 +.global boot_fault_info_el2 +.global boot_fault_info_el3 +boot_fault_info_el1: + .zero 6*8*3 + +.set boot_fault_info_el2, boot_fault_info_el1 + 6*8 +.set boot_fault_info_el3, boot_fault_info_el1 + 2*6*8 .global _start -.section .text +.section .boot.entry,"ax" _start: + ldr x3, =boot_args + stp x0, x1, [x3] + str x2, [x3, #16] + + mrs x1, CurrentEL //get current exception level(CurrentEL EL, bits [3:2]) + lsr x1, x1, #2 + + //early vectors + ldr x2, =boot_vectors_el1 + msr vbar_el1, x2 + + cmp x1, #3 + b.eq 1f + cmp x1, #2 + b.eq 2f + b 3f +1: + ldr x2, =boot_vectors_el2 + msr vbar_el2, x2 + ldr x2, =boot_vectors_el3 + msr vbar_el3, x2 + b 3f +2: + ldr x2, =boot_vectors_el2 + msr vbar_el2, x2 +3: + isb mrs x0, mpidr_el1 and x0, x0, #3 cbz x0, setup_vars @@ -11,8 +53,8 @@ half: b half setup_vars: - ldr x5, =__bss_start - ldr x6, =__bss_end + ldr x5, =__boot_bss_start + ldr x6, =__boot_bss_end clear_bss: cmp x5, x6 b.ge check_hw @@ -45,20 +87,36 @@ check_hw: b .//Halt if none virt: - mov w0, #1 - adrp x1, BOARD_TYPE - strb w0, [x1, #:lo12:BOARD_TYPE] + mov w20, #1 b drop_el3 rpi: - mov w0, #2 - adrp x1, BOARD_TYPE - strb w0, [x1, #:lo12:BOARD_TYPE] + mov w20, #2 b drop_el3 drop_el3: - mrs x0, CurrentEL - lsr x0, x0, #2 - cmp x0, #3 + mrs x9, CurrentEL + lsr x9, x9, #2 + ldr x0, =boot_vectors_el1 + msr vbar_el1, x0 + + cmp x9, #3 + b.eq 1f + cmp x9, #2 + b.eq 2f + b 3f +1: + ldr x0, =boot_vectors_el2 + msr vbar_el2, x0 + ldr x0, =boot_vectors_el3 + msr vbar_el3, x0 + b 3f +2: + ldr x0, =boot_vectors_el2 + msr vbar_el2, x0 +3: + isb + + cmp x9, #3 b.ne el2_entry msr sctlr_el2, xzr @@ -75,20 +133,26 @@ drop_el3: el2_entry: ldr x0, =SCTLR_VALUE_MMU_DISABLED - msr sctlr_el1, x0 + msr sctlr_el1, x0 mrs x0, CurrentEL lsr x0, x0, #2 cmp x0, #2 b.ne stack_setup + ldr x1, =boot_vectors_el2 + msr vbar_el2, x1 + ldr x1, =boot_vectors_el1 + msr vbar_el1, x1 + isb + ldr x0, =CNTHCTL_VALUE msr cnthctl_el2, x0 msr cntvoff_el2, xzr - mov x0, #3 << 20 - msr cpacr_el1, x0 + mov x0, #3 << 20 + msr cpacr_el1, x0 ldr x0, =HCR_RW msr hcr_el2, x0 @@ -102,7 +166,7 @@ el2_entry: eret stack_setup: - ldr x1, =ksp + ldr x1, =boot_stack_top mov sp, x1 mov x29, xzr @@ -110,6 +174,60 @@ stack_setup: mrs x1, CPACR_EL1 orr x1, x1, #(3 << 20) msr CPACR_EL1, x1 - bl kernel_main - - b . \ No newline at end of file + mov x0, x20 + bl boot_mmu_setup + + b . + +//early exc vectors +.section .boot.vectors,"ax" +.align 11 + +.global boot_vectors_el1 +.global boot_vectors_el2 +.global boot_vectors_el3 + +.macro BOOT_VECTOR_COMMON esr, elr, far, info_sym + ldr x0, =\info_sym + mrs x1, \esr + str x1, [x0, #0] + mrs x1, \elr + str x1, [x0, #8] + mrs x1, \far + str x1, [x0, #16] + mov x1, sp + str x1, [x0, #24] + mrs x1, mpidr_el1 ///CPU ID + str x1, [x0, #32] + mrs x1, CurrentEL + str x1, [x0, #40] +1: wfe + b 1b +.endm + +//VBAR EL +// Entry 0 (offset 0x000) +// Entry 1 (offset 0x080) +// Entry 15 (offset 0x780) +//0x800 bytes +.macro VECTOR_TABLE label, esr, elr, far, info +\label: + BOOT_VECTOR_COMMON \esr, \elr, \far, \info + .space 128 - (. - \label) % 128 //pad + BOOT_VECTOR_COMMON \esr, \elr, \far, \info + .space 128 + BOOT_VECTOR_COMMON \esr, \elr, \far, \info + .space 128 + BOOT_VECTOR_COMMON \esr, \elr, \far, \info + .space 128 + .rept 12 + BOOT_VECTOR_COMMON \esr, \elr, \far, \info + .space 128 + .endr +.endm + +VECTOR_TABLE boot_vectors_el1, esr_el1, elr_el1, far_el1, boot_fault_info_el1 +.align 11 +VECTOR_TABLE boot_vectors_el2, esr_el2, elr_el2, far_el2, boot_fault_info_el2 +.align 11 +VECTOR_TABLE boot_vectors_el3, esr_el3, elr_el3, far_el3, boot_fault_info_el3 \ No newline at end of file diff --git a/kernel/dtb.c b/kernel/dtb.c index 2fa644d9..f26943b6 100644 --- a/kernel/dtb.c +++ b/kernel/dtb.c @@ -1,8 +1,8 @@ #include "dtb.h" #include "console/kio.h" #include "std/string.h" +#include "sysregs.h" -#define DTB_ADDR 0x40000000UL #define FDT_MAGIC 0xD00DFEED #define FDT_BEGIN_NODE 0x00000001 @@ -11,6 +11,17 @@ #define FDT_NOP 0x00000004 #define FDT_END 0x00000009 +/* TODO +this file only validates and exposes dtb pointer passed in by the boot, but in boot x0 is supposed to contain the physical address of the dtb in memory +https://docs.kernel.org/arch/arm64/booting.html +on qemu-virt the location depends on the boot mode, for the non elf images the dtb address is passed in x0 +for boots with elf image, the dtb is placed at the start of ram instead, so using x0 isnt correct for the virt boot flow +https://qemu-project.gitlab.io/qemu/system/arm/virt.html +there's also a reported qemu raspi4b issue where no dtb is provided in x0 for -kernel boot and x0 may end up containing 0x100 rather than a dtb pointer +https://gitlab.com/qemu-project/qemu/-/issues/2729 maybe -dtb hw.dtb/dts would fix? it could be tried, but to finish the mem pr quickly ill avoid it (i've already wasted about 10 hours on this) +anyway, the probing policy should be decided in the hw path and not here or in talloc +*/ + struct fdt_header { uint32_t magic; uint32_t totalsize; @@ -26,44 +37,50 @@ struct fdt_header { static struct fdt_header *hdr; +static uint64_t g_dtb_pa = 0; + +void dtb_set_pa(uint64_t dtb_pa) { + if (!dtb_pa) { + g_dtb_pa = 0; + return; + } + if (dtb_pa & HIGH_VA) dtb_pa = VIRT_TO_PHYS(dtb_pa); + g_dtb_pa = dtb_pa; +} + +uint64_t dtb_get_pa() { + return g_dtb_pa; +} + +uintptr_t dtb_base() { + if (!g_dtb_pa) return 0; + uint64_t sctlr = 0; + asm volatile("mrs %0, sctlr_el1" : "=r"(sctlr)); + if ((sctlr & 1) != 0) return PHYS_TO_VIRT(g_dtb_pa); + return (uintptr_t)g_dtb_pa; +} + bool dtb_get_header(){ - if (!hdr) - hdr = (struct fdt_header *)DTB_ADDR; + uintptr_t base = dtb_base(); + if (!base) return false; + hdr = (struct fdt_header *)base; if (__builtin_bswap32(hdr->magic) != FDT_MAGIC) return false; return true; } bool dtb_addresses(uint64_t *start, uint64_t *size){ if (!dtb_get_header()) return false; - *start = (uint64_t)DTB_ADDR; + *start = g_dtb_pa; *size = __builtin_bswap32(hdr->totalsize); return true; } -void dtb_debug_print_all() { - if (!dtb_get_header()) return; - - uint32_t *p = (uint32_t *)(DTB_ADDR + __builtin_bswap32(hdr->off_dt_struct)); - - while (1) { - uint32_t token = __builtin_bswap32(*p++); - if (token == FDT_END) break; - if (token == FDT_BEGIN_NODE) { - const char *name = (const char *)p; - uint32_t skip = 0; - while (((char *)p)[skip]) skip++; - skip = (skip + 4) & ~3; - p += skip / 4; - kprintf(name); - } - } -} - bool dtb_scan(const char *search_name, dtb_node_handler handler, dtb_match_t *match) { if (!dtb_get_header()) return false; - uint32_t *p = (uint32_t *)(DTB_ADDR + __builtin_bswap32(hdr->off_dt_struct)); - const char *strings = (const char *)(DTB_ADDR + __builtin_bswap32(hdr->off_dt_strings)); + uintptr_t base = dtb_base(); + uint32_t *p = (uint32_t *)(base + __builtin_bswap32(hdr->off_dt_struct)); + const char *strings = (const char *)(base + __builtin_bswap32(hdr->off_dt_strings)); int depth = 0; bool active = 0; diff --git a/kernel/dtb.h b/kernel/dtb.h index e4c19d04..bbe16c0f 100644 --- a/kernel/dtb.h +++ b/kernel/dtb.h @@ -13,4 +13,7 @@ typedef struct { typedef int (*dtb_node_handler)(const char *propname, const void *prop, uint32_t len, dtb_match_t *out); bool dtb_addresses(uint64_t *start, uint64_t *size); -bool dtb_scan(const char *search_name, dtb_node_handler handler, dtb_match_t *match); \ No newline at end of file +bool dtb_scan(const char *search_name, dtb_node_handler handler, dtb_match_t *match); +void dtb_set_pa(uint64_t dtb_pa); +uint64_t dtb_get_pa(); +bool dtb_get_header(); \ No newline at end of file diff --git a/kernel/exceptions/exception_handler.c b/kernel/exceptions/exception_handler.c index 7ee66fed..102feb16 100644 --- a/kernel/exceptions/exception_handler.c +++ b/kernel/exceptions/exception_handler.c @@ -5,12 +5,15 @@ #include "timer.h" #include "theme/theme.h" #include "std/string.h" +#include "sysregs.h" static bool panic_triggered = false; void set_exception_vectors(){ extern char exception_vectors[]; - asm volatile ("msr vbar_el1, %0" :: "r"(exception_vectors)); + uintptr_t vbar = (uintptr_t)exception_vectors; + asm volatile ("msr vbar_el1, %0" :: "r"(vbar)); + asm volatile ("isb"); } void handle_exception(const char* type, uint64_t info) { @@ -21,8 +24,10 @@ void handle_exception(const char* type, uint64_t info) { disable_visual();//Disable visual kprintf, since it has additional memory accesses that could be faulting - string s = string_format("%s \r\nESR_EL1: %llx\r\nELR_EL1: %llx\r\nFAR_EL1: %llx",(uintptr_t)type,esr,elr,far); - panic(s.data, info); + char buf[STRING_MAX_LEN];//no heap to avoid corruption + const char *fmt = "%s \r\nESR_EL1: %llx\r\nELR_EL1: %llx\r\nFAR_EL1: %llx"; + string_format_buf(buf, sizeof(buf), fmt,type,esr,elr,far); + panic(buf, info); } void fiq_el1_handler(){ handle_exception("FIQ EXCEPTION\r\n", 0); } @@ -46,8 +51,10 @@ void panic(const char* msg, uint64_t info) { bool old_panic_triggered = panic_triggered; panic_triggered = true; + + const char *title = system_config.panic_text; uart_raw_puts("*** "); - uart_raw_puts(system_config.panic_text); + uart_raw_puts(title); uart_raw_puts(" ***\r\n"); uart_raw_puts(msg); uart_raw_puts("\r\n"); @@ -56,7 +63,10 @@ void panic(const char* msg, uint64_t info) { uart_raw_puts("\r\n"); uart_raw_puts("System Halted\r\n"); if (!old_panic_triggered){ - string s = string_format("%s\r\n%s\r\nError code: %x\r\nSystem Halted",(uint64_t)system_config.panic_text,(uint64_t)msg,info); + char buf[STRING_MAX_LEN]; + const char *fmt = "%s\r\n%s\r\nError code: %llx\r\nSystem Halted"; + size_t len = string_format_buf(buf, sizeof(buf), fmt, title, msg, info); + string s = (string){ .data = buf, .length = (uint32_t)len, .mem_length = 0 }; draw_panic_screen(s); } while (1); diff --git a/kernel/exceptions/exception_vectors_as.S b/kernel/exceptions/exception_vectors_as.S index 546b0f60..c0a35e74 100644 --- a/kernel/exceptions/exception_vectors_as.S +++ b/kernel/exceptions/exception_vectors_as.S @@ -1,27 +1,23 @@ .macro save_ctx msr daifset, #2 - mrs x18, spsr_el1 - lsr x18, x18, #2 - and x18, x18, #0b11 + msr spsel, #1 - cmp x18, #1 + stp x17, x18, [sp, #-16]! + mrs x18, spsr_el1 + and x18, x18, #0xF + cmp x18, #0x5 b.eq 1f - cmp x18, #0 - b.eq 2f - - b 3f - -1: mov x17, sp - b 3f - -2: mrs x17, sp_el0 - ldr x18, =ksp - mov sp, x18 + mrs x17, sp_el0 + b 2f +1: + add x17, sp, #16 +2: + adrp x18, cpec + add x18, x18, :lo12:cpec + ldr x18, [x18] 3: -ldr x18, =cpec -ldr x18, [x18] // Save general-purpose registers x1-x30 stp x0, x1, [x18, #(8 * 0)] stp x2, x3, [x18, #(8 * 2)] @@ -32,6 +28,10 @@ stp x10, x11, [x18, #(8 * 10)] stp x12, x13, [x18, #(8 * 12)] stp x14, x15, [x18, #(8 * 14)] str x16, [x18, #(8 * 16)] +ldr x16, [sp, #0] +str x16, [x18, #(8 * 17)] +ldr x16, [sp, #8] +str x16, [x18, #(8 * 18)] str x19, [x18, #(8 * 19)] stp x20, x21, [x18, #(8 * 20)] stp x22, x23, [x18, #(8 * 22)] @@ -47,6 +47,17 @@ str x17, [x18, #(8 * 31)] mrs x17, spsr_el1 str x17, [x18, #(8 * 33)] +add sp, sp, #16 + +mrs x17, spsr_el1 +lsr x17, x17, #2 +and x17, x17,#0b11 +cmp x17, #0 +b.ne 4f +adrp x18, ksp +add x18, x18, :lo12:ksp +mov sp, x18 +4: .endm .section .vectors, "a", %progbits diff --git a/kernel/exceptions/irq.c b/kernel/exceptions/irq.c index 4be0931d..e22bb293 100644 --- a/kernel/exceptions/irq.c +++ b/kernel/exceptions/irq.c @@ -42,8 +42,8 @@ static void gic_enable_irq(uint32_t irq, uint8_t priority, uint8_t cpu_target) { } void irq_init() { - register_device_memory(GICD_BASE, GICD_BASE); - register_device_memory(GICC_BASE, GICC_BASE); + register_device_memory_dmap(GICD_BASE); + register_device_memory_dmap(GICC_BASE); if (RPI_BOARD != 3){ write32(GICD_BASE, 0); // Disable Distributor @@ -83,6 +83,7 @@ void disable_interrupt(){ void irq_el1_handler() { save_return_address_interrupt(); + mmu_ttbr0_disable_user(); syscall_depth++; uint32_t irq; if (RPI_BOARD == 3){ diff --git a/kernel/filesystem/virtio_9p_pci.cpp b/kernel/filesystem/virtio_9p_pci.cpp index ade23b96..3b877bf4 100644 --- a/kernel/filesystem/virtio_9p_pci.cpp +++ b/kernel/filesystem/virtio_9p_pci.cpp @@ -46,6 +46,7 @@ bool Virtio9PDriver::init(uint32_t partition_sector){ return false; } + np_dev.common_cfg->device_status |= VIRTIO_STATUS_DRIVER_OK; return true; } diff --git a/kernel/filesystem/virtio_9p_pci.hpp b/kernel/filesystem/virtio_9p_pci.hpp index c351c41e..547c95e7 100644 --- a/kernel/filesystem/virtio_9p_pci.hpp +++ b/kernel/filesystem/virtio_9p_pci.hpp @@ -19,7 +19,7 @@ class Virtio9PDriver : public FSDriver { size_t list_contents(const char *path, void* buf, size_t size, uint64_t *offset) override; void close_file(file* descriptor) override; private: - virtio_device np_dev; + virtio_device np_dev = {}; size_t choose_version(); uint32_t open(uint32_t fid); uint32_t attach(); @@ -29,11 +29,11 @@ class Virtio9PDriver : public FSDriver { uint64_t get_attribute(uint32_t fid, uint64_t mask); void p9_max_tag(p9_packet_header* header); void p9_inc_tag(p9_packet_header* header); - size_t max_msize; - uint32_t vfid; - uint32_t mid; + size_t max_msize = 0; + uint32_t vfid = 0; + uint32_t mid = 0; - uint32_t root; + uint32_t root = 0; - chashmap_t *open_files; + chashmap_t *open_files = nullptr; }; \ No newline at end of file diff --git a/kernel/fw/fw_cfg.c b/kernel/fw/fw_cfg.c index 3c9f7706..a0c9e9c6 100644 --- a/kernel/fw/fw_cfg.c +++ b/kernel/fw/fw_cfg.c @@ -28,9 +28,10 @@ struct fw_cfg_dma_access { bool fw_cfg_check(){ if (checked) return true; - register_device_memory(FW_CFG_DATA, FW_CFG_DATA); - checked = read64(FW_CFG_DATA) == 0x554D4551; - if (!checked) mmu_unmap(FW_CFG_DATA, FW_CFG_DATA); + uintptr_t va = PHYS_TO_VIRT(FW_CFG_DATA); + register_device_memory_dmap(va); + checked = read64(va) == 0x554D4551; + if (!checked) mmu_unmap(va, FW_CFG_DATA); return checked; } diff --git a/kernel/gpio.c b/kernel/gpio.c index 417b7855..48ad9f7c 100644 --- a/kernel/gpio.c +++ b/kernel/gpio.c @@ -5,7 +5,7 @@ #include "std/memory_access.h" void reset_gpio(){ - register_device_memory(GPIO_BASE, GPIO_BASE); + register_device_memory_dmap(GPIO_BASE); write32(GPIO_BASE + GPIO_PIN_BASE + 0x94, 0x0); delay(150); } diff --git a/kernel/graph/tres.cpp b/kernel/graph/tres.cpp index 301834ce..80066144 100644 --- a/kernel/graph/tres.cpp +++ b/kernel/graph/tres.cpp @@ -8,6 +8,10 @@ #include "sysregs.h" #include "graphics.h" #include "ui/graphic_types.h" +#include "memory/mmu.h" +#include "memory/page_allocator.h" + +#define WIN_FB_USER_BASE 0x100000000ULL clinkedlist_t *window_list; window_frame *focused_window; @@ -85,9 +89,30 @@ void get_window_ctx(draw_ctx* out_ctx){ clinkedlist_node_t *node = clinkedlist_find(window_list, PHYS_TO_VIRT_P(&p->win_id), (typeof(find_window)*)PHYS_TO_VIRT_P(find_window)); if (node && node->data){ window_frame* frame = (window_frame*)node->data; - if (out_ctx->width && out_ctx->height) - resize_window(out_ctx->width, out_ctx->height); *out_ctx = frame->win_ctx; + + paddr_t phys = (paddr_t)VIRT_TO_PHYS((uintptr_t)frame->win_ctx.fb); + uint64_t size = (uint64_t)frame->win_ctx.width * (uint64_t)frame->win_ctx.height * 4; + size = (size + (GRANULE_4KB - 1)) & ~(GRANULE_4KB - 1); + + if (!p->win_fb_va) p->win_fb_va = WIN_FB_USER_BASE; + + if (p->mm.ttbr0 && size && (p->win_fb_phys != phys || p->win_fb_size != size)) { + if (p->win_fb_size && p->win_fb_phys) { + for (uint64_t off = 0; off < p->win_fb_size; off += GRANULE_4KB) mmu_unmap_table((uint64_t*)p->mm.ttbr0, p->win_fb_va + off, p->win_fb_phys + off); + mm_remove_vma(&p->mm, p->win_fb_va, p->win_fb_va + p->win_fb_size); + } + + for (uint64_t off = 0; off < size; off += GRANULE_4KB) mmu_map_4kb((uint64_t*)p->mm.ttbr0, p->win_fb_va + off, (paddr_t)(phys + off), MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_USER); + mm_add_vma(&p->mm, p->win_fb_va, p->win_fb_va + size, MEM_RW, VMA_KIND_SPECIAL, VMA_FLAG_NOFREE); + + mmu_flush_asid(p->mm.asid); + + p->win_fb_phys = phys; + p->win_fb_size = size; + } + + if (p->mm.ttbr0 && size) out_ctx->fb = (uint32_t*)p->win_fb_va; frame->pid = p->id; } } diff --git a/kernel/hw/hw.c b/kernel/hw/hw.c index bb428d52..92803b34 100644 --- a/kernel/hw/hw.c +++ b/kernel/hw/hw.c @@ -87,21 +87,53 @@ void detect_hardware(){ MSI_OFFSET = 0; LOWEST_ADDR = MMIO_BASE; PM_BASE = MMIO_BASE + 0x100000u; + + //https://www.qemu.org/docs/master/system/arm/raspi.html + //raspi4b has no pcie root + //TODO use DTB as source + #if QEMU + if (BOARD_TYPE != 1) { + PCI_BASE = 0; + XHCI_BASE = 0; + } + #endif } } void hw_high_va(){ if (UART0_BASE) UART0_BASE |= HIGH_VA; if (MMIO_BASE) MMIO_BASE |= HIGH_VA; - if (BOARD_TYPE != 1 && PCI_BASE) - PCI_BASE |= HIGH_VA; + if (PCI_BASE) PCI_BASE |= HIGH_VA; if (GICD_BASE) GICD_BASE |= HIGH_VA; if (GICC_BASE) GICC_BASE |= HIGH_VA; if (MAILBOX_BASE) MAILBOX_BASE |= HIGH_VA; if (SDHCI_BASE) SDHCI_BASE |= HIGH_VA; if (XHCI_BASE) XHCI_BASE |= HIGH_VA; + if (GPIO_BASE) GPIO_BASE |= HIGH_VA; + if (DWC2_BASE) DWC2_BASE |= HIGH_VA; + if (PM_BASE) PM_BASE |= HIGH_VA; } void print_hardware(){ kprintf("Board type %i",BOARD_TYPE); +} + +void hw_mmio_hole_phys(uint64_t *start, uint64_t *end) { + if (!start || !end) return; + *start = 0; + *end = 0; + + if (BOARD_TYPE == 1) { + *start = 0x08000000ULL; + *end = 0x0A000000ULL; + return; + } + + uint64_t base = (uint64_t)MMIO_BASE; + if (base & HIGH_VA) base = VIRT_TO_PHYS(base); + + if (base && base < 0x100000000ULL) { + *start = base; + *end = base + 0x01000000ULL; + } } \ No newline at end of file diff --git a/kernel/hw/hw.h b/kernel/hw/hw.h index dfad19c0..e7cb7b49 100644 --- a/kernel/hw/hw.h +++ b/kernel/hw/hw.h @@ -40,4 +40,5 @@ extern uintptr_t PM_BASE; void detect_hardware(); void print_hardware(); -void hw_high_va(); \ No newline at end of file +void hw_high_va(); +void hw_mmio_hole_phys(uint64_t *start, uint64_t *end); \ No newline at end of file diff --git a/kernel/kernel.c b/kernel/kernel.c index 22a2ba64..a73ea295 100644 --- a/kernel/kernel.c +++ b/kernel/kernel.c @@ -24,10 +24,18 @@ #include "theme/theme.h" #include "tests/test_runner.h" #include "pci/pcie.h" +#include "dtb.h" -void kernel_main() { +extern char __bss_start[]; +extern char __bss_end[]; +void kernel_main(uint64_t board_type, uint64_t dtb_pa) { + for (uintptr_t p = (uintptr_t)__bss_start; p < (uintptr_t)__bss_end; p += 8) *(uint64_t*)p = 0; + BOARD_TYPE = (uint8_t)board_type; + dtb_set_pa(dtb_pa); + if (dtb_get_header()) USE_DTB = 1; detect_hardware(); + hw_high_va(); pre_talloc(); mmu_init(); @@ -66,18 +74,19 @@ void kernel_main() { bool usb_available = can_init_usb ? load_module(&usb_module) : false; bool network_available = false; + bool audio_available = false; if (BOARD_TYPE == 1){ if (system_config.use_net) network_available = load_module(&net_module); - load_module(&audio_module); + audio_available = load_module(&audio_module); } kprint("Kernel initialization finished"); kprint("Starting processes"); - if (BOARD_TYPE == 1) init_audio_mixer(); + if (BOARD_TYPE == 1 && audio_available) init_audio_mixer(); init_filesystem(); diff --git a/kernel/kernel_processes/boot/login_screen.c b/kernel/kernel_processes/boot/login_screen.c index 1d1829ee..120e85cb 100644 --- a/kernel/kernel_processes/boot/login_screen.c +++ b/kernel/kernel_processes/boot/login_screen.c @@ -74,7 +74,7 @@ int login_screen(){ if (key == KEY_ENTER || key == KEY_KPENTER){ if (strcmp(buf,system_config.default_pwd) == 0){ free_sized(buf, 256); - free_sized(s.data,s.mem_length); + string_free(s); sys_set_secure(false); stop_current_process(0); } else @@ -95,7 +95,7 @@ int login_screen(){ old_kp = kp; gpu_flush(); - free_sized(s.data,s.mem_length); + string_free(s); } return 1; } diff --git a/kernel/kernel_processes/kprocess_loader.c b/kernel/kernel_processes/kprocess_loader.c index 4be24740..4386c27f 100644 --- a/kernel/kernel_processes/kprocess_loader.c +++ b/kernel/kernel_processes/kprocess_loader.c @@ -19,12 +19,19 @@ process_t *create_kernel_process(const char *name, int (*func)(int argc, char* a uint64_t stack_size = 0x10000; uintptr_t stack = (uintptr_t)palloc(stack_size, MEM_PRIV_KERNEL, MEM_RW, true); + if (!stack) { + reset_process(proc); + return 0; + } register_allocation(proc->alloc_map, (void*)stack, stack_size); - if (!stack) return 0; uintptr_t heap = (uintptr_t)palloc(PAGE_SIZE, MEM_PRIV_KERNEL, MEM_RW, false); + if (!heap) { + free_registered(proc->alloc_map, (void*)stack); + reset_process(proc); + return 0; + } register_allocation(proc->alloc_map, (void*)heap, PAGE_SIZE); - if (!heap) return 0; proc->stack = (stack + stack_size); proc->stack_size = stack_size; diff --git a/kernel/kernel_processes/windows/launcher.c b/kernel/kernel_processes/windows/launcher.c index 26804f5c..5b3f536c 100644 --- a/kernel/kernel_processes/windows/launcher.c +++ b/kernel/kernel_processes/windows/launcher.c @@ -29,10 +29,10 @@ void *launcher_page = 0; void* alloc_launcher(size_t size){ if (!launcher_page) { - launcher_page = malloc(PAGE_SIZE); + launcher_page = zalloc(PAGE_SIZE); print("Launcher has been given %llx",launcher_page); } - return allocate(launcher_page, size, malloc); + return allocate(launcher_page, size, zalloc); } void add_entry(string_slice name, string_slice ext, string path, package_info info){ @@ -54,10 +54,11 @@ uint16_t find_extension(char *path){ package_info get_pkg_info(char* info_path){ package_info pkg = {}; - char *info = read_full_file(info_path, 0); + size_t info_size = 0; + char *info = read_full_file(info_path, &info_size); if (!info) return pkg; - parse_package_info(info); - // release(info); + pkg = parse_package_info(info); + release(info); return pkg; } @@ -70,7 +71,7 @@ void handle_entry(const char *directory, const char *file) { if (slice_lit_match(ext,"red",true)){ string pkg_info = string_concat(fullpath, string_from_literal("/package.info")); add_entry(name, ext, fullpath, get_pkg_info(pkg_info.data)); - // string_free(pkg_info); + string_free(pkg_info); } } @@ -167,18 +168,24 @@ void activate_current(){ return; } string s = string_format("%s/%v.elf",entry->path.data, entry->name); - file fd = {}; - char *file = read_full_file(s.data, 0); - // string_free(s); + size_t elf_size = 0; + char *file = read_full_file(s.data, &elf_size); + string_free(s); + if (!file || !elf_size) { + rendered_full = false; + return; + } + fb_clear(&ctx, 0); commit_draw_ctx(&ctx); - kprintf("[LAUNCHER] read file %x",fd.size); + kprintf("[LAUNCHER] read file %x", elf_size); disable_interrupt(); - active_proc = load_elf_file(entry->name.data, entry->path.data, file,fd.size); - // release(file); + active_proc = load_elf_file(entry->name.data, entry->path.data, file, elf_size); + release(file); if (!active_proc){ kprintf("[LAUNCHER] Failed to load ELF file"); rendered_full = false; + enable_interrupt(); return; } active_proc->win_id = get_current_proc()->win_id; diff --git a/kernel/linker.ld b/kernel/linker.ld index 738551d2..8e506d9d 100644 --- a/kernel/linker.ld +++ b/kernel/linker.ld @@ -1,12 +1,67 @@ ENTRY(_start) +HIGH_VA = 0xFFFF800000000000; +KERNEL_IMAGE_VA_BASE = 0xFFFFC00000000000; +KIMG_VA = KERNEL_IMAGE_VA_BASE; SECTIONS { . = LOAD_ADDR; + __boot_start = .; + .boot : { + KEEP(*(.boot.entry .boot.entry.*)) + KEEP(*(.boot.vectors .boot.vectors.*)) + *(.boot.text .boot.text.*) + *(.boot.rodata .boot.rodata.*) + *(.boot.data .boot.data.*) + } + + .boot.bss (NOLOAD) : { + . = ALIGN(16); + __boot_bss_start = .; + *(.boot.bss .boot.bss.*) + . = ALIGN(16); + boot_stack = .; + . = . + 0x20000; + boot_stack_top = .; + . = ALIGN(16); + __boot_bss_end = .; + } + + .boot.const : { + . = ALIGN(8); + __boot_kimg_pa_start = .; + QUAD(__kernel_lma) + __boot_kimg_pa_end = .; + QUAD(__kernel_lma_end) + } + . = ALIGN(0x1000); + + __kernel_lma = .; + . = KERNEL_IMAGE_VA_BASE + __kernel_lma; + __kernel_va_start = .; kernel_start = .; - .boot . : { ./build/boot.o(.text) } - .text : { KEEP(*(.text)) } - .data : { *(.data) } - .bss : { + + .text : AT(ADDR(.text) - KERNEL_IMAGE_VA_BASE) { + __text_start = .; + KEEP(*(.text .text.*)) + __text_end = .; + } + . = ALIGN(0x1000); + + .rodata : AT(ADDR(.rodata) - KERNEL_IMAGE_VA_BASE) { + __rodata_start = .; + *(.rodata .rodata.*) + __rodata_end = .; + } + . = ALIGN(0x1000); + + .data : AT(ADDR(.data) - KERNEL_IMAGE_VA_BASE) { + __data_start = .; + *(.data .data.*) + __data_end = .; + } + . = ALIGN(0x1000); + + .bss : AT(ADDR(.bss) - KERNEL_IMAGE_VA_BASE) { . = ALIGN(16); __bss_start = .; *(.bss .bss.* COMMON) @@ -15,10 +70,24 @@ SECTIONS { } . = ALIGN(0x1000); - .vectors : { KEEP(*(.vectors)) } + .kstack (NOLOAD) : AT(ADDR(.kstack) - KERNEL_IMAGE_VA_BASE) { + . = ALIGN(16); + __kstack_bottom = .; + . = . + 0x10000; + ksp = .; + __kstack_top = .; + } + . = ALIGN(0x1000); + + .vectors : AT(ADDR(.vectors) - KERNEL_IMAGE_VA_BASE) { + __vectors_start = .; + KEEP(*(.vectors)) + __vectors_end = .; + } . = ALIGN(16); - ksp = . + 0x10000; + kcode_end = .; - kcode_end = . + 0x10000; + __kernel_va_end = .; + __kernel_lma_end = __kernel_va_end - KERNEL_IMAGE_VA_BASE; } \ No newline at end of file diff --git a/kernel/memory/addr.c b/kernel/memory/addr.c new file mode 100644 index 00000000..ae0716c1 --- /dev/null +++ b/kernel/memory/addr.c @@ -0,0 +1,30 @@ +#include "addr.h" + +bool kva_is_dmap(kaddr_t va){ + return ((uint64_t)va & HIGH_VA) == HIGH_VA; +} + +kaddr_t dmap_pa_to_kva(paddr_t pa){ + if (!pa) return 0; + return (kaddr_t)PHYS_TO_VIRT((uintptr_t)pa); +} + +paddr_t dmap_kva_to_pa(kaddr_t va){ + if (!va) return 0; + return (paddr_t)VIRT_TO_PHYS((uintptr_t)va); +} + +void* pt_pa_to_va(paddr_t pa){ + uint64_t sctlr = 0; + asm volatile("mrs %0, sctlr_el1" : "=r"(sctlr)); + if ((sctlr & 1) == 0) return (void*)(uintptr_t)pa; + return (void*)(uintptr_t)dmap_pa_to_kva(pa); +} + +paddr_t pt_va_to_pa(const void* va){ + uintptr_t v = (uintptr_t)va; + uint64_t sctlr = 0; + asm volatile("mrs %0, sctlr_el1" : "=r"(sctlr)); + if ((sctlr & 1) == 0) return (paddr_t)v; + return dmap_kva_to_pa((kaddr_t)v); +} diff --git a/kernel/memory/addr.h b/kernel/memory/addr.h new file mode 100644 index 00000000..07cce4cc --- /dev/null +++ b/kernel/memory/addr.h @@ -0,0 +1,17 @@ +#pragma once + +#include "types.h" +#include "sysregs.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool kva_is_dmap(kaddr_t va); +kaddr_t dmap_pa_to_kva(paddr_t pa); +paddr_t dmap_kva_to_pa(kaddr_t va); +void* pt_pa_to_va(paddr_t pa); +paddr_t pt_va_to_pa(const void* va); +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/kernel/memory/mm_process.c b/kernel/memory/mm_process.c new file mode 100644 index 00000000..c7be8b4c --- /dev/null +++ b/kernel/memory/mm_process.c @@ -0,0 +1,232 @@ +#include "process/process.h" +#include "memory/mmu.h" +#include "memory/addr.h" +#include "std/memory.h" +#include "memory/mm_process.h" + +vma* mm_find_vma(mm_struct *mm, uaddr_t va){ + if (!mm) return 0; + for (uint16_t i = 0; i < mm->vma_count; i++){ + vma *m = &mm->vmas[i]; + if (va < m->start) return 0; + if (va < m->end) return m; + } + return 0; +} + +bool mm_add_vma(mm_struct *mm, uaddr_t start, uaddr_t end, uint8_t prot, uint8_t kind, uint8_t flags){ + if (!mm) return false; + start &= ~(PAGE_SIZE - 1); + end = (end + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + if (start >= end) return false; + if (mm->vma_count >= MAX_VMAS) return false; + + uint16_t ins = 0; + while (ins < mm->vma_count && mm->vmas[ins].start < start) ins++; + + if (ins > 0 && start < mm->vmas[ins - 1].end) return false; + if (ins < mm->vma_count && end > mm->vmas[ins].start) return false; + + for (uint16_t i = mm->vma_count; i > ins; i--) mm->vmas[i] = mm->vmas[i - 1]; + + mm->vmas[ins] = (vma){start, end, prot, kind, flags}; + mm->vma_count++; + + if (ins > 0) { + vma *a = &mm->vmas[ins - 1]; + vma *b = &mm->vmas[ins]; + if (a->end == b->start && a->prot == b->prot && a->kind == b->kind && a->flags == b->flags && !(a->flags & VMA_FLAG_USERALLOC)) { + a->end = b->end; + for (uint16_t i = ins; i + 1 < mm->vma_count; i++) mm->vmas[i] = mm->vmas[i + 1]; + mm->vma_count--; + ins--; + } + } + + if (ins + 1 < mm->vma_count) { + vma *a = &mm->vmas[ins]; + vma *b = &mm->vmas[ins + 1]; + if (a->end == b->start && a->prot == b->prot && a->kind == b->kind && a->flags == b->flags && !(a->flags & VMA_FLAG_USERALLOC)) { + a->end = b->end; + for (uint16_t i = ins + 1; i + 1 < mm->vma_count; i++)mm->vmas[i] = mm->vmas[i + 1]; + mm->vma_count--; + } + } + return true; +} + +bool mm_remove_vma(mm_struct *mm, uaddr_t start, uaddr_t end) { + if (!mm) return false; + start &= ~(PAGE_SIZE - 1); + end = (end + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + if (start >= end) return false; + + for (uint16_t i = 0; i < mm->vma_count; i++) { + vma *m = &mm->vmas[i]; + if (end <= m->start) return false; + if (start >= m->end) continue; + + bool track_free = m->kind != VMA_KIND_HEAP && m->kind != VMA_KIND_STACK; + uaddr_t free_start = 0; + uaddr_t free_end = 0; + + if (start <= m->start && end >= m->end) { + free_start = m->start; + free_end = m->end; + for (uint16_t j = i; j + 1 < mm->vma_count; j++) mm->vmas[j] = mm->vmas[j + 1]; + mm->vma_count--; + } else if (start <= m->start) { + free_start = m->start; + free_end = end; + m->start = end; + } else if (end >= m->end) { + free_start = start; + free_end = m->end; + m->end = start; + } else { + if (mm->vma_count >= MAX_VMAS) return false; + free_start = start; + free_end = end; + for (uint16_t j = mm->vma_count; j > i + 1; j--) mm->vmas[j] = mm->vmas[j-1]; + mm->vmas[i + 1] = (vma){end, m->end, m->prot, m->kind, m->flags}; + m->end = start; + mm->vma_count++; + } + + if (!track_free) return true; + + free_start &= ~(PAGE_SIZE - 1); + free_end = (free_end + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + if (free_start >= free_end) return true; + + uint16_t ins = 0; + while (ins < mm->mmap_free_count && mm->mmap_free[ins].start < free_start) ins++; + + if (ins > 0 && mm->mmap_free[ins - 1].end >= free_start) { + if (mm->mmap_free[ins - 1].end < free_end) mm->mmap_free[ins - 1].end = free_end; + ins--; + } else { + if (mm->mmap_free_count >= MAX_VMAS) return true; + for (uint16_t j = mm->mmap_free_count; j > ins; j--) mm->mmap_free[j] = mm->mmap_free[j - 1]; + mm->mmap_free[ins] = (mm_free_range){free_start, free_end}; + mm->mmap_free_count++; + } + + while (ins + 1 < mm->mmap_free_count && mm->mmap_free[ins].end >= mm->mmap_free[ins + 1].start) { + if (mm->mmap_free[ins].end < mm->mmap_free[ins + 1].end) mm->mmap_free[ins].end = mm->mmap_free[ins + 1].end; + for (uint16_t j = ins + 1; j + 1 < mm->mmap_free_count; j++) mm->mmap_free[j] = mm->mmap_free[j + 1]; + mm->mmap_free_count--; + } + + while (mm->mmap_free_count) { + mm_free_range *top = &mm->mmap_free[mm->mmap_free_count - 1]; + if (top->end != mm->mmap_cursor) break; + mm->mmap_cursor = top->start; + mm->mmap_free_count--; + } + + return true; + } + return false; +} + +uaddr_t mm_alloc_mmap(mm_struct *mm, size_t size, uint8_t prot, uint8_t kind, uint8_t flags) { + if (!mm) return 0; + if (!size) return 0; + size = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + + for (uint16_t i = 0; i < mm->mmap_free_count; i++) { + mm_free_range *r = &mm->mmap_free[i]; + size_t span = r->end - r->start; + if(span < size) continue; + uaddr_t base = (r->end - size) & ~(PAGE_SIZE - 1); + if (base < r->start) continue; + if (!mm_add_vma(mm, base, base + size, prot, kind, flags)) return 0; + if (base == r->start && base + size == r->end) { + for (uint16_t j = i; j + 1 < mm->mmap_free_count; j++) mm->mmap_free[j] = mm->mmap_free[j + 1]; + mm->mmap_free_count--; + } else if (base == r->start) r->start += size; + else r->end = base; + return base; + } + + uaddr_t heap_guard = mm->brk + (MM_GAP_PAGES * PAGE_SIZE); + if (!mm->mmap_cursor) return 0; + uaddr_t base = (mm->mmap_cursor - size) & ~(PAGE_SIZE - 1); + if (base < heap_guard) return 0; + if (base + size > mm->mmap_top) return 0; + if (!mm_add_vma(mm, base, base + size, prot, kind, flags)) return 0; + mm->mmap_cursor = base; + return base; +} + +bool mm_try_handle_page_fault(process_t *proc, uintptr_t far, uint64_t esr) { + if (!proc || !proc->mm.ttbr0) return false; + + uint64_t ec = (esr >> 26) & 0x3F; + uint32_t iss = esr & 0xFFFFFF; + uint8_t ifsc = esr & 0x3F; + + bool is_exec = ec == 0x20; + bool is_write = false; + if (!is_exec) is_write = ((iss >> 6) & 1) != 0; + + if (ifsc >= 0x9 && ifsc <= 0xB) { + if (!mmu_set_access_flag((uint64_t*)proc->mm.ttbr0, far)) return false; + mmu_flush_asid(proc->mm.asid); + return true; + } + + if (ifsc >= 0xD && ifsc <= 0xF) return false; + if (ifsc < 0x4 || ifsc > 0x7) return false; + + uintptr_t va_page = far & ~(PAGE_SIZE-1); + vma *m = mm_find_vma(&proc->mm, va_page); + if (!m) return false; + if ((is_exec && !(m->prot & MEM_EXEC)) || (is_write && !(m->prot & MEM_RW))) return false; + + + if (!(m->flags & VMA_FLAG_DEMAND)) return false; + + if (m->kind == VMA_KIND_STACK) { + uintptr_t sp = proc->sp & ~(PAGE_SIZE - 1); + uintptr_t low = sp > (32 * PAGE_SIZE) ? sp - (32 * PAGE_SIZE) : proc->mm.stack_limit; + if (va_page < low || va_page < proc->mm.stack_limit || va_page >= proc->mm.stack_top) return false; + uintptr_t grow_to = proc->mm.stack_commit; + if (va_page < grow_to) grow_to = va_page; + if (((proc->mm.stack_top - grow_to) / PAGE_SIZE) > proc->mm.cap_stack_pages) return false; + for (uintptr_t page = proc->mm.stack_commit - PAGE_SIZE; page >= grow_to; page -= PAGE_SIZE) { + paddr_t phys = palloc_inner(PAGE_SIZE, MEM_PRIV_USER, MEM_RW, true, false); + if (!phys) { + for (uintptr_t undo = page + PAGE_SIZE; undo < proc->mm.stack_commit; undo += PAGE_SIZE) { + uint64_t pa = 0; + if (!mmu_unmap_and_get_pa((uint64_t*)proc->mm.ttbr0, undo, &pa)) continue; + pfree((void*)dmap_pa_to_kva((paddr_t)pa), PAGE_SIZE); + if (proc->mm.rss_stack_pages) proc->mm.rss_stack_pages--; + } + return false; + } + memset((void*)dmap_pa_to_kva(phys), 0, PAGE_SIZE); + mmu_map_4kb((uint64_t*)proc->mm.ttbr0, page, phys, MAIR_IDX_NORMAL, m->prot | MEM_NORM, MEM_PRIV_USER); + proc->mm.rss_stack_pages++; + if (page == 0) break; + } + proc->mm.stack_commit = grow_to; + mmu_flush_asid(proc->mm.asid); + return true; + } + + if ((m->kind == VMA_KIND_HEAP && proc->mm.rss_heap_pages >= proc->mm.cap_heap_pages) || (m->kind == VMA_KIND_ANON && proc->mm.rss_anon_pages >= proc->mm.cap_anon_pages)) return false; + + paddr_t phys = palloc_inner(PAGE_SIZE, MEM_PRIV_USER, MEM_RW, true, false); + if (!phys) return false; + + if (m->kind != VMA_KIND_ANON || (m->flags & VMA_FLAG_ZERO)) memset((void*)dmap_pa_to_kva(phys), 0, PAGE_SIZE); + mmu_map_4kb((uint64_t*)proc->mm.ttbr0, va_page, phys, MAIR_IDX_NORMAL, m->prot | MEM_NORM, MEM_PRIV_USER); + mmu_flush_asid(proc->mm.asid); + + if (m->kind == VMA_KIND_HEAP) proc->mm.rss_heap_pages++; + else if (m->kind == VMA_KIND_ANON) proc->mm.rss_anon_pages++; + + return true; +} diff --git a/kernel/memory/mm_process.h b/kernel/memory/mm_process.h new file mode 100644 index 00000000..3fb97796 --- /dev/null +++ b/kernel/memory/mm_process.h @@ -0,0 +1,63 @@ +#pragma once + +#include "types.h" +#include "memory/page_allocator.h" + +typedef struct process process_t; + +#define VMA_FLAG_DEMAND 1 +#define VMA_FLAG_USERALLOC 2 +#define VMA_FLAG_ZERO 4 +#define VMA_FLAG_NOFREE 8 +#define VMA_KIND_ELF 1 +#define VMA_KIND_HEAP 2 +#define VMA_KIND_STACK 3 +#define VMA_KIND_ANON 4 +#define VMA_KIND_SPECIAL 5 + +#define MAX_VMAS 32 +#define MM_GAP_PAGES 16 + +typedef struct vma { + uaddr_t start; + uaddr_t end; + uint8_t prot; + uint8_t kind; + uint8_t flags; +} vma; + +typedef struct mm_free_range { + uaddr_t start; + uaddr_t end; +} mm_free_range; + +typedef struct mm_struct { + uintptr_t *ttbr0; + paddr_t ttbr0_phys; + uint16_t asid; + uint32_t asid_gen; + vma vmas[MAX_VMAS]; + uint16_t vma_count; + mm_free_range mmap_free[MAX_VMAS]; + uint16_t mmap_free_count; + uaddr_t heap_start; + uaddr_t brk; + uaddr_t brk_max; + uaddr_t mmap_top; + uaddr_t mmap_cursor; + uaddr_t stack_top; + uaddr_t stack_limit; + uaddr_t stack_commit; + uint64_t rss_heap_pages; + uint64_t rss_stack_pages; + uint64_t rss_anon_pages; + uint64_t cap_heap_pages; + uint64_t cap_stack_pages; + uint64_t cap_anon_pages; +} mm_struct; + +vma* mm_find_vma(mm_struct *mm, uaddr_t va); +bool mm_add_vma(mm_struct *mm, uaddr_t start, uaddr_t end, uint8_t prot, uint8_t kind, uint8_t flags); +bool mm_remove_vma(mm_struct *mm, uaddr_t start, uaddr_t end); +uaddr_t mm_alloc_mmap(mm_struct *mm, size_t size, uint8_t prot, uint8_t kind, uint8_t flags); +bool mm_try_handle_page_fault(process_t *proc, uintptr_t far, uint64_t esr); \ No newline at end of file diff --git a/kernel/memory/mmu.c b/kernel/memory/mmu.c index 6f49239e..d1b53b4a 100644 --- a/kernel/memory/mmu.c +++ b/kernel/memory/mmu.c @@ -11,26 +11,91 @@ #include "process/process.h" #include "process/scheduler.h" #include "sysregs.h" +#include "memory/addr.h" #include "std/memory.h" +#include "exceptions/exception_handler.h" #include "alloc/mem_types.h" +#include "memory/talloc.h" +#include "memory/va_layout.h" + +extern char kcode_end; + +extern char __text_start[]; +extern char __text_end[]; +extern char __rodata_start[]; +extern char __rodata_end[]; +extern char __data_start[]; +extern char __data_end[]; +extern char __bss_start[]; +extern char __bss_end[]; +extern char __vectors_start[]; +extern char __vectors_end[]; +extern char __kstack_bottom[]; +extern char __kstack_top[]; + +static uintptr_t *kernel_ttbr0; +static uintptr_t *kernel_ttbr1; +static uint64_t kernel_ttbr0_hw; +static bool ttbr0_user_on; -#define PD_TABLE 0b11 -#define PD_BLOCK 0b01 +static bool mmu_verbose; +static inline void mmu_flush_icache(); +static inline void mmu_flush_all(); + + +static uint64_t asid_shift; +static uint16_t asid_mask; + +static uint32_t asid_gen; +static uint32_t asid_max; +static uint32_t asid_next; +static uint64_t asid_used[65536/64]; + +static uint64_t pttbr_hw; +static uint16_t pttbr_asid; +static uintptr_t *pttbr; + +static inline uint64_t make_pte(uint64_t pa, uint64_t attr_index, uint8_t mem_attr, uint8_t level, uint64_t type) { + uint64_t sh = (mem_attr & MEM_DEV) ? 0 : 0b11; + uint64_t ap = 0; + + if (level == MEM_PRIV_KERNEL) ap = (mem_attr & MEM_RW) ? 0 : 0b10; + else ap = (mem_attr & MEM_RW) ? 1 : 0b11; + + uint64_t attr = PTE_AF | (sh << PTE_SH_SHIFT) | (ap << PTE_AP_SHIFT) | (attr_index << PTE_ATTR_SHIFT); + + if (level == MEM_PRIV_KERNEL) { + attr |= PTE_UXN; + if (!(mem_attr & MEM_EXEC)) attr |= PTE_PXN; + } else if (level == MEM_PRIV_USER) { + attr |= PTE_NG; + attr |= PTE_PXN; + if (!(mem_attr & MEM_EXEC)) attr |= PTE_UXN; + } else if (level == MEM_PRIV_SHARED) { + attr |= PTE_NG; + attr |= PTE_PXN; + if (!(mem_attr & MEM_EXEC)) attr |= PTE_UXN; + } -#define UXN_BIT 54 -#define PXN_BIT 53 -#define AF_BIT 10 -#define SH_BIT 8 -#define AP_BIT 6 -#define MAIR_BIT 2 + return (pa & PTE_ADDR_MASK) | attr | type; +} -#define PAGE_TABLE_ENTRIES 512 +static uint64_t *walk_or_alloc(uint64_t *table, uint64_t index, int level, uint64_t va) { + uint64_t e = table[index]; -#define ADDR_MASK 0xFFFFFFFFF000ULL + if (!(e & 1)){ + uint64_t *n = mmu_alloc(); + table[index] = (pt_va_to_pa(n) & PTE_ADDR_MASK) | PD_TABLE; + return n; + } -uint64_t *kernel_mmu_page; + if ((e & 0b11) != PD_TABLE){ + kprintf("[mmu] *walk_or_alloc bad type l=%d va=%llx idx=%llu e=%llx", level, (uint64_t)va, (uint64_t)index, (uint64_t)e); + panic("mmu *walk_or_alloc bad type", va); + } -static bool mmu_verbose; + return (uint64_t*)pt_pa_to_va(e & PTE_ADDR_MASK); +} void mmu_enable_verbose(){ mmu_verbose = true; @@ -43,30 +108,68 @@ void mmu_enable_verbose(){ }\ }) -void mmu_map_2mb(uint64_t *table, uint64_t va, uint64_t pa, uint64_t attr_index) { +void mmu_map_2mb(uint64_t *table, uint64_t va, uint64_t pa, uint64_t attr_index, uint8_t mem_attr, uint8_t level) { uint64_t l0_index = (va >> 39) & 0x1FF; uint64_t l1_index = (va >> 30) & 0x1FF; uint64_t l2_index = (va >> 21) & 0x1FF; - kprintfv("[MMU] Mapping 2mb memory %x at [%i][%i][%i] for EL1", va, l0_index,l1_index,l2_index); + if (!table) panic("mmu_map_2mb null root", va); + kprintfv("[MMU] Mapping 2mb memory %llx at [%i][%i][%i] for EL%i", (uint64_t)va, (int)l0_index, (int)l1_index, (int)l2_index, (int)level); - if (!(table[l0_index] & 1)) { - uint64_t* l1 = (uint64_t*)talloc(PAGE_SIZE); - table[l0_index] = ((uint64_t)l1 & ADDR_MASK) | PD_TABLE; + if ((va & (GRANULE_2MB - 1ULL)) || (pa & (GRANULE_2MB - 1ULL))) { + kprintf("[mmu] map2 align va=%llx pa=%llx", (uint64_t)va, (uint64_t)pa); + panic("mmu_map_2mb unaligned", va); } - uint64_t* l1 = (uint64_t*)(table[l0_index] & ADDR_MASK); + uint64_t* l1 = walk_or_alloc(table, l0_index, 0, va); + uint64_t* l2 = walk_or_alloc(l1, l1_index, 1, va); - if (!(l1[l1_index] & 1)) { - uint64_t* l2 = (uint64_t*)talloc(PAGE_SIZE); - l1[l1_index] = ((uint64_t)l2 & ADDR_MASK) | PD_TABLE; + uint64_t want = make_pte(pa, attr_index, mem_attr, level, PD_BLOCK); + uint64_t old = l2[l2_index]; + + if ((old & 1) == 0){ + l2[l2_index] = want; + return; } - uint64_t* l2 = (uint64_t*)(l1[l1_index] & ADDR_MASK); - - //For now we make this not executable. We'll need to to separate read_write, read_only and executable sections - uint64_t attr = ((uint64_t)1 << UXN_BIT) | ((uint64_t)0 << PXN_BIT) | (1 << AF_BIT) | (0b11 << SH_BIT) | (0b00 << AP_BIT) | (attr_index << MAIR_BIT) | PD_BLOCK; - l2[l2_index] = (pa & ADDR_MASK) | attr; + if ((old & 0b11) == PD_BLOCK){ + if ((old & PTE_ADDR_MASK) == (want & PTE_ADDR_MASK)){ + uint64_t diff = (want & ~PTE_AF) | (old & PTE_AF); + l2[l2_index] = diff; + return; + } + kprintf("[mmu] map2 conflict va=%llx old=%llx newpa=%llx root=%llx", (uint64_t)va, (uint64_t)old, (uint64_t)pa, (uint64_t)table); + panic("mmu_map_2mb remap conflict", va); + } + + if ((old & 0b11) == PD_TABLE){ + uint64_t* l3 = (uint64_t*)pt_pa_to_va(old & PTE_ADDR_MASK); + uint64_t base = pa; + uint64_t attr = (want & ~PTE_ADDR_MASK) & ~0b11; + + for (uint64_t i = 0; i < PAGE_TABLE_ENTRIES; i++) { + uint64_t p = base + (i * GRANULE_4KB); + uint64_t e = l3[i]; + uint64_t expect = (p & PTE_ADDR_MASK) | attr | PD_TABLE; + + if ((e & 1ULL) == 0) { + l3[i] = expect; + continue; + } + + uint64_t diff = (e ^ expect) & ~(PTE_ADDR_MASK | PTE_AF); + + if (diff != 0 || (e & PTE_ADDR_MASK) != (expect & PTE_ADDR_MASK)) { + kprintf("[mmu] map2 table mismatch va=%llx i=%llu e=%llx", (uint64_t)va, (uint64_t)i, (uint64_t)e); + panic("mmu_map_2mb table mismatch", va); + } + } + + return; + } + + kprintf("[mmu] map2 bad type va=%llx old=%llx", (uint64_t)va, (uint64_t)old); + panic("mmu_map_2mb bad type", va); } //Level 0 = EL0, Level 1 = EL1, Level 2 = Shared @@ -76,57 +179,122 @@ void mmu_map_4kb(uint64_t *table, uint64_t va, uint64_t pa, uint64_t attr_index, uint64_t l2_index = (va >> 21) & 0x1FF; uint64_t l3_index = (va >> 12) & 0x1FF; - if (!(table[l0_index] & 1)) { - uint64_t* l1 = (uint64_t*)talloc(PAGE_SIZE); - table[l0_index] = ((uint64_t)l1 & ADDR_MASK) | PD_TABLE; - } - - uint64_t* l1 = (uint64_t*)(table[l0_index] & ADDR_MASK); - if (!(l1[l1_index] & 1)) { - uint64_t* l2 = (uint64_t*)talloc(PAGE_SIZE); - l1[l1_index] = ((uint64_t)l2 & ADDR_MASK) | PD_TABLE; - } - - uint64_t* l2 = (uint64_t*)(l1[l1_index] & ADDR_MASK); + if (!table) panic("mmu_map_4kb null root", va); + if ((table == (uint64_t*)kernel_ttbr0 || table == (uint64_t*)kernel_ttbr1) && level == MEM_PRIV_USER) panic("mmu_map_4kb user map in kernel ttbr", va); + + pa &= ~(GRANULE_4KB - 1ULL); + va &= ~(GRANULE_4KB - 1ULL); + + uint8_t permission = 0; + if (level == MEM_PRIV_KERNEL) permission = (mem_attributes & MEM_RW) ? 0 : 0b10; + else if (level == MEM_PRIV_SHARED) permission = (mem_attributes & MEM_RW) ? 1 : 0b11; + else permission = (mem_attributes & MEM_RW) ? 1 : 0b11; + + //TODO: proper memory permissions, including accounting for WXN + kprintfv("[MMU] Mapping 4kb memory %llx at [%i][%i][%i][%i] for EL%i = %llx | %llx permission: %i", + (uint64_t)va, (int)l0_index, (int)l1_index, (int)l2_index, (int)l3_index, (int)level, (uint64_t)pa, + (uint64_t)make_pte(pa, attr_index, mem_attributes, level, PD_TABLE), (int)permission); + + uint64_t* l1 = walk_or_alloc(table, l0_index, 0, va); + uint64_t* l2 = walk_or_alloc(l1, l1_index, 1, va); + uint64_t l2_val = l2[l2_index]; if (!(l2_val & 1)) { - uint64_t* l3 = (uint64_t*)talloc(PAGE_SIZE); - l2[l2_index] = ((uint64_t)l3 & ADDR_MASK) | PD_TABLE; + uint64_t* l3 = mmu_alloc(); + l2[l2_index] = (pt_va_to_pa(l3) & PTE_ADDR_MASK) | PD_TABLE; + l2_val = l2[l2_index]; } else if ((l2_val & 0b11) == PD_BLOCK){ - uart_puts("[MMU error]: Region not mapped for address "); - uart_puthex(va); - uart_puts("already mapped at higher granularity\n"); - return; + uint64_t base = (l2_val & PTE_ADDR_MASK) & ~(GRANULE_2MB - 1); + uint64_t expected = base + (va & (GRANULE_2MB - 1)); + uint64_t want = make_pte(pa, attr_index, mem_attributes, level, PD_TABLE); + + uint64_t old_attr = (l2_val & ~PTE_ADDR_MASK) & ~0b11; + uint64_t want_attr = (want & ~PTE_ADDR_MASK) & ~0b11; + + if (expected == pa &&old_attr == want_attr) return; + + { + uint64_t old = l2[l2_index]; + if ((old & 0b11) != PD_BLOCK){ + kprintf("[mmu] split expected block va=%llx l2=%llu e=%llx", (uint64_t)va, (uint64_t)l2_index, (uint64_t)old); + panic("mmu_split not a block", va); + } + + uint64_t base2 = (old & PTE_ADDR_MASK) & ~(GRANULE_2MB - 1); + uint64_t* l3 = mmu_alloc(); + uint64_t attr = (old & ~PTE_ADDR_MASK) & ~0b11; + + for (uint64_t i = 0; i < PAGE_TABLE_ENTRIES; i++){ + uint64_t p = base2 + (i * GRANULE_4KB); + l3[i] = (p & PTE_ADDR_MASK) | attr | PD_TABLE; + } + + l2[l2_index] = (pt_va_to_pa(l3) & PTE_ADDR_MASK) | PD_TABLE; + } + + l2_val = l2[l2_index]; } - - uint64_t* l3 = (uint64_t*)(l2[l2_index] & ADDR_MASK); - - if (l3[l3_index] & 1){ - uart_puts("[MMU warning]: Section already mapped "); - uart_puthex(va); - uart_puts(" "); - uart_puthex((uintptr_t)table); - uart_putc('\n'); - return; + + if ((l2_val & 0b11) != PD_TABLE){ + kprintf("[mmu] l2 bad type va=%llx e=%llx", (uint64_t)va, (uint64_t)l2_val); + panic("mmu_map_4kb l2 bad type", va); } - - uint8_t permission = 0; - - //TODO: proper memory permissions, including accounting for WXN - switch (level) - { - case MEM_PRIV_USER: permission = 0b01; break; - case MEM_PRIV_SHARED: permission = mem_attributes & MEM_EXEC ? 0b11 : 0b01; break; - case MEM_PRIV_KERNEL: permission = 0b00; break; - - default: - break; + + uint64_t *l3 = (uint64_t*)pt_pa_to_va(l2_val & PTE_ADDR_MASK); + + uint64_t want = make_pte(pa, attr_index, mem_attributes, level, PD_TABLE); + uint64_t old = l3[l3_index]; + + if (old & 1){ + if ((old & 0b11) != PD_TABLE){ + kprintf("[mmu] remap non-page va=%llx old=%llx", (uint64_t)va, (uint64_t)old); + panic("mmu_map_4kb remap non-page", va); + } + + if ((old & PTE_ADDR_MASK) == (want & PTE_ADDR_MASK)){ + uint64_t diff = (old ^ want) & ~(PTE_ADDR_MASK | PTE_AF); + if (diff == 0) return; + + uint64_t rs = get_user_ram_start(); + uint64_t re = get_user_ram_end(); + + bool kernel_root = table == (uint64_t*)kernel_ttbr0 || table == (uint64_t*)kernel_ttbr1; + bool in_ram = pa >= rs && pa < re; + + uint64_t apmask = (0x3ULL << PTE_AP_SHIFT); + + if (kernel_root && in_ram){ + if (diff & apmask){ + uint64_t forced = old & ~apmask; + if ((old & apmask) != 0){ + l3[l3_index] = forced; + mmu_flush_all(); + mmu_flush_icache(); + } + return; + } + + uint64_t allowed = 0; + allowed |= (0x7ULL << PTE_ATTR_SHIFT); + allowed |= (0x3ULL << PTE_SH_SHIFT); + allowed |= PTE_PXN | PTE_UXN; + + if (!(diff & ~allowed)) { + l3[l3_index] = want; + mmu_flush_all(); + mmu_flush_icache(); + return; + } + } + } + + kprintf("[mmu] remap conflict va=%llx old=%llx want=%llx newpa=%llx ttbr=%llx idx=%llu,%llu,%llu,%llu", + (uint64_t)va, (uint64_t)old, (uint64_t)want, (uint64_t)pa, (uint64_t)table, + (uint64_t)l0_index, (uint64_t)l1_index, (uint64_t)l2_index, (uint64_t)l3_index); + panic("mmu_map_4kb remap conflict", va); } - //TODO: tests for permissions - uint64_t attr = ((uint64_t)(level == MEM_PRIV_KERNEL) << UXN_BIT) | ((uint64_t)(level == MEM_PRIV_USER) << PXN_BIT) | (1 << AF_BIT) | (0b01 << SH_BIT) | (permission << AP_BIT) | (attr_index << MAIR_BIT) | 0b11; - kprintfv("[MMU] Mapping 4kb memory %x at [%i][%i][%i][%i] for EL%i = %x | %x permission: %i", va, l0_index,l1_index,l2_index,l3_index,level,pa,attr,permission); - - l3[l3_index] = (pa & ADDR_MASK) | attr; + + l3[l3_index] = want; } static inline void mmu_flush_all() { @@ -135,6 +303,7 @@ static inline void mmu_flush_all() { "tlbi vmalle1is\n" // Invalidate all EL1 TLB entries (Inner Shareable) "dsb ish\n" // Ensure completion of TLB invalidation "isb\n" // Synchronize pipeline + ::: "memory" ); } @@ -142,241 +311,704 @@ static inline void mmu_flush_icache() { asm volatile ( "ic iallu\n" // Invalidate all instruction caches to PoU "isb\n" // Ensure completion before continuing + ::: "memory" ); } uintptr_t* mmu_default_ttbr(){ - return kernel_mmu_page; + return kernel_ttbr1; } -void mmu_unmap_table(uintptr_t *table, uintptr_t va, uintptr_t pa){ +void mmu_unmap_table(uint64_t *table, uint64_t va, uint64_t pa){ uint64_t l0_index = (va >> 39) & 0x1FF; uint64_t l1_index = (va >> 30) & 0x1FF; uint64_t l2_index = (va >> 21) & 0x1FF; uint64_t l3_index = (va >> 12) & 0x1FF; - kprintfv("[MMU] Unmapping 4kb memory %x at [%i][%i][%i][%i] for EL1", va, l0_index,l1_index,l2_index, l3_index); - if (!(table[l0_index] & 1)) return; - - uint64_t* l1 = (uint64_t*)(table[l0_index] & ADDR_MASK); - if (!(l1[l1_index] & 1)) return; - - uint64_t* l2 = (uint64_t*)(l1[l1_index] & ADDR_MASK); - uint64_t l3_val = l2[l2_index]; - if (!(l3_val & 1)) return; - else if ((l3_val & 0b11) == PD_BLOCK){ - l2[l2_index] = 0; - return; + if (!table) panic("mmu_unmap null root", va); + kprintfv("[MMU] Unmapping 4kb memory %llx at [%i][%i][%i][%i] for EL1", (uint64_t)va, (int)l0_index, (int)l1_index, (int)l2_index, (int)l3_index); + + va &= ~(GRANULE_4KB - 1); + pa &= ~(GRANULE_4KB - 1); + + uint64_t e0 = table[l0_index]; + if (!(e0 & 1)) return; + if ((e0 & 0b11) != PD_TABLE) panic("mmu_unmap l0 bad type", va); + + uint64_t* l1 = (uint64_t*)pt_pa_to_va(e0 & PTE_ADDR_MASK); + uint64_t e1 = l1[l1_index]; + if (!(e1 & 1)) return; + if ((e1 & 0b11) != PD_TABLE) panic("mmu_unmap l1 bad type", va); + + uint64_t* l2 = (uint64_t*)pt_pa_to_va(e1 & PTE_ADDR_MASK); + uint64_t e2 = l2[l2_index]; + if (!(e2 & 1)) return; + + if ((e2 & 0b11) == PD_BLOCK) { + uint64_t old = l2[l2_index]; + if ((old & 0b11) != PD_BLOCK){ + kprintf("[mmu] split expected block va=%llx l2=%llu e=%llx", (uint64_t)va, (uint64_t)l2_index, (uint64_t)old); + panic("mmu_split not a block", va); + } + + uint64_t base = (old & PTE_ADDR_MASK) & ~(GRANULE_2MB - 1); + uint64_t *l3 = mmu_alloc(); + uint64_t attr = (old & ~PTE_ADDR_MASK) & ~0b11; + + for (uint64_t i = 0; i < PAGE_TABLE_ENTRIES; i++){ + uint64_t p = base + (i * GRANULE_4KB); + l3[i] = (p & PTE_ADDR_MASK) | attr | PD_TABLE; + } + + l2[l2_index] = (pt_va_to_pa(l3) & PTE_ADDR_MASK) | PD_TABLE; + + e2 = l2[l2_index]; + if (!(e2 & 1)) panic("mmu_unmap split vanished", va); + } + + if ((e2 & 0b11) != PD_TABLE) panic("mmu_unmap l2 bad type", va); + + uint64_t *l3 = (uint64_t*)pt_pa_to_va(e2 & PTE_ADDR_MASK); + uint64_t old = l3[l3_index]; + if (!(old & 1)) return; + + if ((old & 0b11) != PD_TABLE) { + kprintf("[mmu] unmap non-page va=%llx old=%llx", (uint64_t)va, (uint64_t)old); + panic("mmu_unmap non-page", va); + } + + if ((old & PTE_ADDR_MASK) != (pa & PTE_ADDR_MASK)) { + kprintf("[mmu] unmap pa mismatch va=%llx old=%llx want=%llx", (uint64_t)va, (uint64_t)(old & PTE_ADDR_MASK), (uint64_t)(pa & PTE_ADDR_MASK)); + panic("mmu_unmap pa mismatch", va); } - - uint64_t* l3 = (uint64_t*)(l2[l2_index] & ADDR_MASK); l3[l3_index] = 0; +} + +void mmu_unmap(uint64_t va, uint64_t pa){ + mmu_unmap_table((uint64_t*)kernel_ttbr1, va, pa); + //if (pttbr) mmu_unmap_table((uint64_t*)pttbr, va, pa); mmu_flush_all(); mmu_flush_icache(); } -void mmu_unmap(uint64_t va, uint64_t pa){ - mmu_unmap_table(kernel_mmu_page, va, pa); - mmu_unmap_table(pttbr, va, pa); -} uint64_t *mmu_alloc(){ - return (uint64_t*)talloc(PAGE_SIZE); + uint64_t* p = (uint64_t*)talloc(GRANULE_4KB); + if (!p) panic("mmu_alloc out of memory", 0); + memset(p, 0, GRANULE_4KB); + return p; } -extern uintptr_t cpec; -extern uintptr_t ksp; - -extern void mmu_start(uint64_t *mmu); +extern void mmu_start(uint64_t *ttbr1, uint64_t *ttbr0); uintptr_t heap_end; void mmu_init() { - kernel_mmu_page = mmu_alloc(); - uintptr_t kstart = mem_get_kmem_start(); - uintptr_t kend = mem_get_kmem_end(); - uintptr_t heapstart = get_user_ram_start(); + uint64_t mmfr0 = 0; + asm volatile("mrs %0, id_aa64mmfr0_el1" : "=r"(mmfr0)); + uint64_t asidbits = (mmfr0 >> 4) & 0xF; + asid_shift = 48; + asid_mask = (asidbits >= 2) ? 0xFFFF : 0xFF; + asid_gen = 1; + asid_max = (uint32_t)asid_mask + 1; + asid_next = 1; + memset(asid_used, 0, sizeof(asid_used)); + asid_used[0] = 1; + + kernel_ttbr0 = (uintptr_t*)mmu_alloc(); + kernel_ttbr1 = (uintptr_t*)mmu_alloc(); + uintptr_t kimg_base = KERNEL_IMAGE_VA_BASE; + uintptr_t kstart_pa = ((uintptr_t)__text_start) - kimg_base; + uintptr_t kend_pa = ((uintptr_t)&kcode_end) - kimg_base; + uintptr_t pa0 = kstart_pa & ~(GRANULE_4KB - 1); + uintptr_t pa1 = (kend_pa + (GRANULE_4KB - 1)) & ~(GRANULE_4KB - 1); + + for (uintptr_t pa = pa0; pa < pa1; pa += GRANULE_4KB) mmu_map_4kb((uint64_t*)kernel_ttbr1, pa | HIGH_VA, pa, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_KERNEL); + + uintptr_t v0 = ((uintptr_t)__text_start) & ~(GRANULE_4KB - 1); + uintptr_t v1 = (((uintptr_t)__text_end) + (GRANULE_4KB - 1)) & ~(GRANULE_4KB - 1); + for (uintptr_t va = v0; va < v1; va += GRANULE_4KB) mmu_map_4kb((uint64_t*)kernel_ttbr1, va, va - kimg_base, MAIR_IDX_NORMAL, MEM_EXEC | MEM_NORM, MEM_PRIV_KERNEL); + + v0 = ((uintptr_t)__vectors_start) & ~(GRANULE_4KB - 1); + v1 = (((uintptr_t)__vectors_end) + (GRANULE_4KB - 1)) & ~(GRANULE_4KB - 1); + for (uintptr_t va = v0; va < v1; va += GRANULE_4KB) mmu_map_4kb((uint64_t*)kernel_ttbr1, va, va - kimg_base, MAIR_IDX_NORMAL, MEM_EXEC | MEM_NORM, MEM_PRIV_KERNEL); + + v0 = ((uintptr_t)__rodata_start) & ~(GRANULE_4KB - 1); + v1 = (((uintptr_t)__rodata_end) + (GRANULE_4KB - 1)) & ~(GRANULE_4KB - 1); + for (uintptr_t va = v0; va < v1; va += GRANULE_4KB) mmu_map_4kb((uint64_t*)kernel_ttbr1, va, va - kimg_base, MAIR_IDX_NORMAL, MEM_NORM, MEM_PRIV_KERNEL); + + v0 = ((uintptr_t)__data_start) & ~(GRANULE_4KB - 1); + v1 = (((uintptr_t)__data_end) + (GRANULE_4KB - 1)) & ~(GRANULE_4KB - 1); + for (uintptr_t va = v0; va < v1; va += GRANULE_4KB) mmu_map_4kb((uint64_t*)kernel_ttbr1, va, va - kimg_base, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_KERNEL); + + v0 = ((uintptr_t)__bss_start) & ~(GRANULE_4KB - 1); + v1 = (((uintptr_t)__bss_end) + (GRANULE_4KB - 1)) & ~(GRANULE_4KB - 1); + for (uintptr_t va = v0; va < v1; va += GRANULE_4KB) mmu_map_4kb((uint64_t*)kernel_ttbr1, va, va - kimg_base, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_KERNEL); + + v0 = ((uintptr_t)__kstack_bottom) & ~(GRANULE_4KB - 1); + v1 = (((uintptr_t)__kstack_top) + (GRANULE_4KB - 1)) & ~(GRANULE_4KB - 1); + for (uintptr_t va = v0; va < v1; va += GRANULE_4KB) mmu_map_4kb((uint64_t*)kernel_ttbr1, va, va - kimg_base, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_KERNEL); + + uint64_t dstart = 0; + uint64_t dsize = 0; + if (dtb_addresses(&dstart,&dsize)) { + uint64_t dend = (dstart + dsize + (GRANULE_4KB - 1)) & ~(GRANULE_4KB - 1); + for (uint64_t addr = dstart & ~(GRANULE_4KB - 1); addr < dend; addr += GRANULE_4KB) { + //mmu_map_4kb((uint64_t*)kernel_ttbr0, addr, addr, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_KERNEL); + mmu_map_4kb((uint64_t*)kernel_ttbr1, addr | HIGH_VA, addr, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_KERNEL); + } + } - for (uint64_t addr = kstart; addr < kend; addr += GRANULE_2MB) - mmu_map_2mb(kernel_mmu_page, addr, addr, MAIR_IDX_NORMAL); - - for (uint64_t addr = heapstart; addr < heap_end; addr += GRANULE_4KB) - mmu_map_4kb(kernel_mmu_page, addr, addr, MAIR_IDX_NORMAL, MEM_DEV | MEM_RW, MEM_PRIV_KERNEL); - - mmu_map_2mb(kernel_mmu_page, (uintptr_t)kernel_mmu_page, (uintptr_t)kernel_mmu_page, MAIR_IDX_DEVICE); + uint64_t ram_start = get_user_ram_start(); + uint64_t ram_end = get_user_ram_end(); + uint64_t mmio_skip_start = 0; + uint64_t mmio_skip_end = 0; + hw_mmio_hole_phys(&mmio_skip_start, &mmio_skip_end); + + for (uint64_t pa = ram_start; pa < ram_end;) { + if (mmio_skip_end && pa >= mmio_skip_start && pa < mmio_skip_end) { + pa = mmio_skip_end; + continue; + } + if ((!(pa & (GRANULE_2MB - 1))) && (ram_end - pa) >= GRANULE_2MB){ + //mmu_map_2mb((uint64_t*)kernel_ttbr0, pa, pa, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_KERNEL); + mmu_map_2mb((uint64_t*)kernel_ttbr1, pa | HIGH_VA, pa, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_KERNEL); + pa += GRANULE_2MB; + } else { + //mmu_map_4kb((uint64_t*)kernel_ttbr0, pa, pa, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_KERNEL); + mmu_map_4kb((uint64_t*)kernel_ttbr1, pa | HIGH_VA, pa, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_KERNEL); + pa += GRANULE_4KB; + } + } - uint64_t dstart; - uint64_t dsize; - if (dtb_addresses(&dstart,&dsize)) - for (uint64_t addr = dstart; addr <= dstart + dsize; addr += GRANULE_4KB) - mmu_map_4kb(kernel_mmu_page, addr, addr, MAIR_IDX_NORMAL, MEM_RO, MEM_PRIV_KERNEL); + if (BOARD_TYPE == 1) { + for (uint64_t pa = 0x08000000ULL; pa < 0x0A000000ULL; pa += GRANULE_2MB) mmu_map_2mb((uint64_t*)kernel_ttbr1, pa | HIGH_VA, pa, MAIR_IDX_DEVICE, MEM_RW | MEM_DEV, MEM_PRIV_KERNEL); + } else { + if (RPI_BOARD == 4) for (uint64_t pa = 0xFE000000ULL; pa < 0xFF000000ULL; pa += GRANULE_2MB) mmu_map_2mb((uint64_t*)kernel_ttbr1, pa | HIGH_VA, pa, MAIR_IDX_DEVICE, MEM_RW | MEM_DEV, MEM_PRIV_KERNEL); + else for (uint64_t pa = 0x3F000000ULL; pa < 0x40000000ULL; pa += GRANULE_2MB) mmu_map_2mb((uint64_t*)kernel_ttbr1, pa | HIGH_VA, pa, MAIR_IDX_DEVICE, MEM_RW | MEM_DEV, MEM_PRIV_KERNEL); + } - hw_high_va(); + if (PCI_BASE && (PCI_BASE < (mmio_skip_start | HIGH_VA) || PCI_BASE >= (mmio_skip_end | HIGH_VA))) { + uint64_t p = VIRT_TO_PHYS(PCI_BASE) & ~(GRANULE_2MB - 1); + //mmu_map_2mb((uint64_t*)kernel_ttbr0, p, p, MAIR_IDX_DEVICE, MEM_RW | MEM_DEV, MEM_PRIV_KERNEL); + mmu_map_2mb((uint64_t*)kernel_ttbr1, p | HIGH_VA, p, MAIR_IDX_DEVICE, MEM_RW | MEM_DEV, MEM_PRIV_KERNEL); + } - mmu_start(kernel_mmu_page); + if (XHCI_BASE && (XHCI_BASE < (mmio_skip_start | HIGH_VA) || XHCI_BASE >= (mmio_skip_end | HIGH_VA))) { + uint64_t p = VIRT_TO_PHYS(XHCI_BASE) & ~(GRANULE_2MB - 1); + //mmu_map_2mb((uint64_t*)kernel_ttbr0, p, p, MAIR_IDX_DEVICE, MEM_RW | MEM_DEV, MEM_PRIV_KERNEL); + mmu_map_2mb((uint64_t*)kernel_ttbr1, p | HIGH_VA, p, MAIR_IDX_DEVICE, MEM_RW | MEM_DEV, MEM_PRIV_KERNEL); + } + //hw_high_va(); + //mmu_start((uint64_t*)kernel_ttbr1, (uint64_t*)kernel_ttbr0); + + //kernel_ttbr0 = (uintptr_t*)PHYS_TO_VIRT((uintptr_t)kernel_ttbr0); + //kernel_ttbr1 = (uintptr_t*)PHYS_TO_VIRT((uintptr_t)kernel_ttbr1); + + talloc_enable_high_va(); + page_alloc_enable_high_va(); + kernel_ttbr0_hw = pt_va_to_pa(kernel_ttbr0) & PTE_ADDR_MASK; + + uint64_t ttbr1_pa = pt_va_to_pa(kernel_ttbr1) & PTE_ADDR_MASK; + asm volatile("msr ttbr1_el1, %0" :: "r"(ttbr1_pa)); + asm volatile("dsb ish"); + asm volatile("isb"); + + mmu_swap_ttbr(0); + + mmu_flush_all(); + mmu_flush_icache(); // kprintf("Finished MMU init"); } void mmu_copy(uintptr_t *new_ttbr, uintptr_t *old_ttbr, int level){ + if (!new_ttbr || !old_ttbr) return; for (int i = 0; i < PAGE_TABLE_ENTRIES; i++){ - if (old_ttbr[i] & 1){ - if (level == 3 || (level == 2 && ((old_ttbr[i] & 0b11) == PD_BLOCK))){ - new_ttbr[i] = old_ttbr[i]; - } else { - uintptr_t *old_entry = (uintptr_t*)(old_ttbr[i] & ADDR_MASK); - uintptr_t *new_entry = mmu_alloc(); - if (!old_entry || !new_entry) continue; - uint64_t entry = old_ttbr[i] & ~(ADDR_MASK); - new_ttbr[i] = entry | ((uintptr_t)new_entry & ADDR_MASK); - mmu_copy(new_entry, old_entry, level+1); - } + if (!(old_ttbr[i] & 1)) continue; + + if (level == 3 || (level == 2 && ((old_ttbr[i] & 0b11) == PD_BLOCK))){ + new_ttbr[i] = old_ttbr[i]; + continue; } - } -} -void mmu_free_ttbr_l(uintptr_t *ttbr, int level){ - for (int i = 0; i < PAGE_TABLE_ENTRIES; i++){ - if (ttbr[i] & 1){ - if (level == 3 || (level == 2 && ((ttbr[i] & 0b11) == PD_BLOCK))) continue; - uintptr_t *entry = (uintptr_t*)(ttbr[i] & ADDR_MASK); - mmu_free_ttbr_l(entry, level+1); + if ((old_ttbr[i] & 0b11) != PD_TABLE){ + kprintf("[mmu] copy bad type lvl=%d i=%d e=%llx", level, i, (uint64_t)old_ttbr[i]); + panic("mmu_copy bad type", (uint64_t)old_ttbr); } + + uintptr_t *old_entry = (uintptr_t*)pt_pa_to_va(old_ttbr[i] & PTE_ADDR_MASK); + uintptr_t *new_entry = (uintptr_t*)mmu_alloc(); + uint64_t entry = old_ttbr[i] & ~PTE_ADDR_MASK; + new_ttbr[i] = entry | (pt_va_to_pa(new_entry) & PTE_ADDR_MASK); + mmu_copy(new_entry, old_entry, level+1); } - temp_free(ttbr, PAGE_SIZE); } -void mmu_map_all(uintptr_t pa){ - process_t *processes = get_all_processes(); - for (int i = 0; i < MAX_PROCS; i++){ - if (processes[i].state != STOPPED && processes[i].ttbr){ - mmu_map_2mb(processes[i].ttbr, pa, pa, MAIR_IDX_DEVICE); - } - } +typedef struct { + uintptr_t *table; + int level; + int i; +} mmu_free_frame_t; + +void mmu_map_all(paddr_t pa){ + if (!kernel_ttbr1) return; + uintptr_t base = pa & ~(GRANULE_2MB - 1); + + //mmu_map_2mb((uint64_t*)kernel_ttbr0, base, base, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_KERNEL); + mmu_map_2mb((uint64_t*)kernel_ttbr1, base | HIGH_VA, base, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_KERNEL); + mmu_flush_all(); mmu_flush_icache(); } void mmu_free_ttbr(uintptr_t *ttbr){ - mmu_free_ttbr_l(ttbr, 0); + if (!ttbr) return; + + mmu_free_frame_t stack[4]; + int sp = 0; + + stack[sp++] = (mmu_free_frame_t){ttbr,0,0}; + + while (sp > 0){ + mmu_free_frame_t *f = &stack[sp -1]; + + if (f->i >= PAGE_TABLE_ENTRIES){ + temp_free(f->table, GRANULE_4KB); + sp--; + continue; + } + + uintptr_t e = f->table[f->i++]; + if (!(e & 1)) continue; + if (f->level == 3) continue; + if (f->level == 2 && ((e & 0b11) == PD_BLOCK)) continue; + if ((e & 0b11) != PD_TABLE) continue; + + uintptr_t *child = (uintptr_t*)pt_pa_to_va(e & PTE_ADDR_MASK); + + if (sp >= 4){ + kprintf("[mmu] free_ttbr stack overflow lvl=%d e=%llx", f->level, (uint64_t)e); + panic("mmu_free_ttbr stack overflow", (uintptr_t)child); + } + + stack[sp++] = (mmu_free_frame_t){ child, f->level + 1, 0 }; + } } uintptr_t* mmu_new_ttbr(){ - uintptr_t *ttbr = mmu_alloc(); - mmu_copy(ttbr, kernel_mmu_page,0); - return ttbr; + return (uintptr_t*)mmu_alloc(); } -void register_device_memory(uint64_t va, uint64_t pa){ - if (pttbr && pttbr != kernel_mmu_page)//TODO: This won't be necessary once kernel is exclusively in ttbr1 - mmu_map_4kb(pttbr, va, pa, MAIR_IDX_DEVICE, MEM_RW, MEM_PRIV_KERNEL); - mmu_map_4kb(kernel_mmu_page, va, pa, MAIR_IDX_DEVICE, MEM_RW, MEM_PRIV_KERNEL); +void register_device_memory(kaddr_t va, paddr_t pa){ + uint64_t phys = (uint64_t)pa; + uint64_t vlow = va; + if (((vlow >> 47) & 1ULL) == 0) vlow = phys | HIGH_VA; + mmu_map_4kb((uint64_t*)kernel_ttbr1, vlow, phys, MAIR_IDX_DEVICE, MEM_RW | MEM_DEV, MEM_PRIV_KERNEL); + //mmu_map_4kb((uint64_t*)kernel_ttbr1, vhigh, phys, MAIR_IDX_DEVICE, MEM_RW | MEM_DEV, MEM_PRIV_KERNEL); + //if (pttbr) mmu_map_4kb((uint64_t*)pttbr, vlow, phys, MAIR_IDX_DEVICE, MEM_RW | MEM_DEV, MEM_PRIV_KERNEL); + mmu_flush_all(); mmu_flush_icache(); } -void register_device_memory_2mb(uint64_t va, uint64_t pa){ - if (pttbr && pttbr != kernel_mmu_page)//TODO: This won't be necessary once kernel is exclusively in ttbr1 - mmu_map_2mb(pttbr, va, pa, MAIR_IDX_DEVICE); - mmu_map_2mb(kernel_mmu_page, va, pa, MAIR_IDX_DEVICE); +void register_device_memory_dmap(kaddr_t va) { + register_device_memory(va, kva_is_dmap(va) ? dmap_kva_to_pa(va) : (paddr_t)va); +} + +void register_device_memory_2mb(kaddr_t va, paddr_t pa){ + uint64_t phys = ((uint64_t)pa) & ~(GRANULE_2MB - 1); + uint64_t vlow = va; + if (((vlow >> 47) & 1ULL) == 0) vlow = phys | HIGH_VA; + vlow &= ~(GRANULE_2MB - 1); + + //mmu_map_2mb((uint64_t*)kernel_ttbr0, vlow, phys, MAIR_IDX_DEVICE, MEM_RW | MEM_DEV, MEM_PRIV_KERNEL); + mmu_map_2mb((uint64_t*)kernel_ttbr1, vlow, phys, MAIR_IDX_DEVICE, MEM_RW | MEM_DEV, MEM_PRIV_KERNEL); + //if (pttbr) mmu_map_2mb((uint64_t*)pttbr, vlow, phys, MAIR_IDX_DEVICE, MEM_RW | MEM_DEV, MEM_PRIV_KERNEL); + mmu_flush_all(); mmu_flush_icache(); } -void register_proc_memory(uint64_t va, uint64_t pa, uint8_t attributes, uint8_t level){ - if (pttbr && pttbr != kernel_mmu_page) - mmu_map_4kb(pttbr, va, pa, MAIR_IDX_NORMAL, attributes, level); - mmu_map_4kb(kernel_mmu_page, va, pa, MAIR_IDX_NORMAL, attributes, level); +void register_proc_memory(uint64_t va, paddr_t pa, uint8_t attributes, uint8_t level){ + uint64_t phys = (uint64_t)pa; + + if (level == MEM_PRIV_USER){ + if (!pttbr) panic("register_proc_memory no pttbr for user", va); + mmu_map_4kb((uint64_t*)pttbr, va, phys, MAIR_IDX_NORMAL, attributes | MEM_NORM, level); + mmu_flush_asid(pttbr_asid); + mmu_flush_icache(); + return; + } + + uint64_t vlow = va; + if (((vlow >> 47) & 1ULL) == 0) vlow = phys | HIGH_VA; + + mmu_map_4kb((uint64_t*)kernel_ttbr1, vlow, phys, MAIR_IDX_NORMAL, attributes | MEM_NORM, level); + + //if (pttbr && ((va >> 47) & 1ULL) == 0) mmu_map_4kb((uint64_t*)pttbr, vlow, phys, MAIR_IDX_NORMAL, attributes | MEM_NORM, level); + mmu_flush_all(); mmu_flush_icache(); } -uintptr_t mmu_translate(uintptr_t va){ - uint64_t *table = pttbr && (va >> 48) == 0 ? pttbr : kernel_mmu_page; +uintptr_t mmu_translate(uint64_t *root, uintptr_t va, int *status){ + int dummy; + if (!status) status = &dummy; + + if (((uint64_t)va >> 48) != ((0-((((uint64_t)va >> 47)&1))) & 0xFFFF)) { + *status = MMU_TR_ERR_PARAM; + return 0; + } + + if (!root) { + if (((va >> 47) & 1) != 0) root = (uint64_t*)kernel_ttbr1; + else root = (uint64_t*)(pttbr ? pttbr : kernel_ttbr0); + } + + if (!root){ + *status = MMU_TR_ERR_PARAM; + return 0; + } uint64_t l0_index = (va >> 39) & 0x1FF; uint64_t l1_index = (va >> 30) & 0x1FF; uint64_t l2_index = (va >> 21) & 0x1FF; uint64_t l3_index = (va >> 12) & 0x1FF; - if (!(table[l0_index] & 1)) { + uint64_t e0 = root[l0_index]; + if ((e0 & 1) == 0){ + *status = MMU_TR_ERR_L1; kprintfv("L1 Table missing"); return 0; } - uint64_t* l1 = (uint64_t*)(table[l0_index] & ADDR_MASK); - if (!(l1[l1_index] & 1)) { + if ((e0 & 0b11) != PD_TABLE){ + *status = MMU_TR_ERR_L1; + kprintfv("L1 Table missing"); + return 0; + } + + uint64_t* l1 = (uint64_t*)pt_pa_to_va(e0 & PTE_ADDR_MASK); + uint64_t e1 = l1[l1_index]; + if ((e1 & 1) == 0){ + *status = MMU_TR_ERR_L2; kprintfv("L2 Table missing"); return 0; } - uint64_t* l2 = (uint64_t*)(l1[l1_index] & ADDR_MASK); - uint64_t l3_val = l2[l2_index]; - if (!(l3_val & 1)) { + if ((e1 & 0b11) != PD_TABLE){ + *status = MMU_TR_ERR_L2; + kprintfv("L2 Table missing"); + return 0; + } + uint64_t* l2 = (uint64_t*)pt_pa_to_va(e1 & PTE_ADDR_MASK); + uint64_t e2 = l2[l2_index]; + if (!(e2 & 1)) { + *status = MMU_TR_ERR_L3; kprintfv("L3 Table missing"); return 0; } - if (!((l3_val >> 1) & 1)){ - return l3_val & ADDR_MASK; + if ((e2 & 0b11) == PD_BLOCK){ + *status = MMU_TR_OK; + return (uintptr_t)(((e2 & PTE_ADDR_MASK) & ~(GRANULE_2MB - 1ULL)) | ((uint64_t)va & (GRANULE_2MB - 1ULL))); } - uint64_t* l3 = (uint64_t*)(l2[l2_index] & ADDR_MASK); - uint64_t l4_val = l3[l3_index]; - if (!(l4_val & 1)){ + if ((e2 & 0b11) != PD_TABLE){ + *status = MMU_TR_ERR_L3; + kprintfv("L3 Table missing"); + return 0; + } + + uint64_t* l3 = (uint64_t*)pt_pa_to_va(e2 & PTE_ADDR_MASK); + uint64_t e3 = l3[l3_index]; + if ((e3 & 1) == 0){ + *status = MMU_TR_ERR_L4; + kprintfv("L4 Table entry missing"); + return 0; + } + if ((e3 & 0b11ULL) != PD_TABLE){ + *status = MMU_TR_ERR_L4; kprintfv("L4 Table entry missing"); return 0; } - return l4_val & ADDR_MASK; + + *status = MMU_TR_OK; + return (uintptr_t)((e3 & PTE_ADDR_MASK) | ((uint64_t)va & (GRANULE_4KB - 1))); } void debug_mmu_address(uint64_t va){ + int tr = 0; + uintptr_t pa = mmu_translate(0, (uintptr_t)va, &tr); - uint64_t *table = pttbr ? pttbr : kernel_mmu_page; + uart_raw_puts("[mmu dbg] VA="); + uart_puthex(va); + uart_raw_puts(" PA="); + uart_puthex(pa); + uart_raw_puts(" ST="); + uart_puthex((uintptr_t)tr); + uart_raw_putc('\n'); uint64_t l0_index = (va >> 39) & 0x1FF; uint64_t l1_index = (va >> 30) & 0x1FF; uint64_t l2_index = (va >> 21) & 0x1FF; uint64_t l3_index = (va >> 12) & 0x1FF; + uint64_t *table; + if (((va >> 47) & 1) != 0) table = (uint64_t*)kernel_ttbr1; + else table = (uint64_t*)(pttbr ? pttbr : kernel_ttbr0); kprintf("Address %llx is meant to be mapped to [%i][%i][%i][%i]",va, l0_index,l1_index,l2_index,l3_index); - if (!(table[l0_index] & 1)) { + if (!table) { + kprintf("L1 Table missing"); + return; + } + + uint64_t e0 = table[l0_index]; + if (!(e0 & 1) || ((e0 & 0b11) != PD_TABLE)) { kprintf("L1 Table missing"); return; } - uint64_t* l1 = (uint64_t*)(table[l0_index] & ADDR_MASK); - if (!(l1[l1_index] & 1)) { + + uint64_t* l1 = (uint64_t*)pt_pa_to_va(e0 & PTE_ADDR_MASK); + uint64_t e1 = l1[l1_index]; + if (!(e1 & 1) || ((e1 & 0b11) != PD_TABLE)) { kprintf("L2 Table missing"); return; } - uint64_t* l2 = (uint64_t*)(l1[l1_index] & ADDR_MASK); - uint64_t l3_val = l2[l2_index]; - if (!(l3_val & 1)) { + + uint64_t* l2 = (uint64_t*)pt_pa_to_va(e1 & PTE_ADDR_MASK); + uint64_t e2 = l2[l2_index]; + if (!(e2 & 1)){ kprintf("L3 Table missing"); return; } - if (!((l3_val >> 1) & 1)){ + if ((e2 & 0b11) == PD_BLOCK) { kprintf("Mapped as 2MB memory in L3"); - kprintf("Entry: %b", l3_val); + kprintf("Entry: %b", (uint64_t)e2); return; } - uint64_t* l3 = (uint64_t*)(l2[l2_index] & ADDR_MASK); - uint64_t l4_val = l3[l3_index]; - if (!(l4_val & 1)){ + if ((e2 & 0b11) != PD_TABLE) { + kprintf("L3 Table missing"); + return; + } + + uint64_t* l3 = (uint64_t*)pt_pa_to_va(e2 & PTE_ADDR_MASK); + uint64_t e3 = l3[l3_index]; + if (!(e3 & 1)){ kprintf("L4 Table entry missing"); return; } - kprintf("Entry: %b", l4_val); + kprintf("Entry: %b", e3); return; } -extern void mmu_swap(uintptr_t* ttbr); +void mmu_ttbr0_disable_user() { + asm volatile("msr ttbr0_el1, %0" :: "r"((uint64_t)kernel_ttbr0_hw)); + asm volatile("dsb ish\n\tisb" ::: "memory"); + ttbr0_user_on = false; +} + +void mmu_ttbr0_enable_user() { + uint64_t hw = kernel_ttbr0_hw; + if (pttbr && pttbr != (uintptr_t*)kernel_ttbr0) hw = pttbr_hw; + asm volatile("msr ttbr0_el1, %0" :: "r"(hw)); + asm volatile("dsb ish\n\tisb" ::: "memory"); + ttbr0_user_on = pttbr && pttbr != (uintptr_t*)kernel_ttbr0; +} + +bool mmu_ttbr0_user_enabled() { + return ttbr0_user_on; +} + +void mmu_swap_ttbr(mm_struct *mm){ + if (mm && mm->ttbr0) { + pttbr = mm->ttbr0; + pttbr_asid = mm->asid & asid_mask; + pttbr_hw = ((uint64_t)pttbr_asid << asid_shift) | (mm->ttbr0_phys & PTE_ADDR_MASK); + } else { + pttbr = (uintptr_t*)kernel_ttbr0; + pttbr_asid = 0; + pttbr_hw = kernel_ttbr0_hw; + } + mmu_ttbr0_disable_user(); +} + +void mmu_flush_asid(uint16_t asid) { + uint64_t v = (uint64_t)(asid & asid_mask) << asid_shift; + asm volatile("dsb ishst" ::: "memory"); + asm volatile("tlbi aside1is, %0":: "r"(v) : "memory"); + asm volatile("dsb ish\n\tisb" ::: "memory"); +} + +void mmu_asid_ensure(mm_struct *mm) { + if (!mm) return; + if (!asid_max) return; + if (mm->asid && mm->asid_gen == asid_gen) return; + + for (;;) { + uint32_t scanned = 0; + while (scanned + 1 < asid_max) { + if (asid_next >= asid_max) asid_next = 1; + + uint32_t a = asid_next++; + scanned++; -uintptr_t *pttbr; + uint32_t w = a >> 6; + uint32_t b = a & 63; + if (asid_used[w] & (1ULL << b)) continue; + + asid_used[w] |= 1ULL << b; + mm->asid = (uint16_t)a; + mm->asid_gen = asid_gen; + return; + } + + asid_gen++; + mmu_flush_all(); + memset(asid_used, 0, sizeof(asid_used)); + asid_used[0] = 1; + asid_next = 1; + } +} + +void mmu_asid_release(mm_struct *mm){ + if (!mm || !mm->asid) return; + if (!asid_max) return; + if (mm->asid_gen != asid_gen) { + mm->asid = 0; + mm->asid_gen = 0; + return; + } -void mmu_swap_ttbr(uintptr_t* ttbr){ - pttbr = ttbr ? ttbr : kernel_mmu_page; + uint32_t a = (uint32_t)(mm->asid & asid_mask); + uint32_t w = a >> 6; + uint32_t b = a & 63; + mmu_flush_asid(mm->asid); + asid_used[w] &= ~(1ULL << b); + mm->asid = 0; + mm->asid_gen = 0; } + +bool mmu_unmap_and_get_pa(uint64_t *table, uint64_t va, uint64_t *pa) { + if (!table) return false; + va &= ~(GRANULE_4KB-1); + + uint64_t l0_index = (va >> 39) & 0x1FF; + uint64_t l1_index = (va >> 30) & 0x1FF; + uint64_t l2_index = (va >> 21) & 0x1FF; + uint64_t l3_index = (va >> 12) & 0x1FF; + + uint64_t l1_val = table[l0_index]; + if (!(l1_val & 1)) return false; + if ((l1_val & 0b11) != PD_TABLE) return false; + uint64_t* l1 = (uint64_t*)pt_pa_to_va(l1_val & PTE_ADDR_MASK); + + uint64_t l2_val = l1[l1_index]; + if (!(l2_val & 1)) return false; + if ((l2_val & 0b11) != PD_TABLE) return false; + uint64_t* l2 = (uint64_t*)pt_pa_to_va(l2_val & PTE_ADDR_MASK); + + uint64_t l3_val = l2[l2_index]; + if (!(l3_val & 1)) return false; + + if ((l3_val & 0b11) == PD_BLOCK) { + uint64_t base = l3_val & PTE_ADDR_MASK; + uint64_t off = va & (GRANULE_2MB - 1); + uint64_t p = (base & ~(GRANULE_2MB - 1)) + off; + + if (pa) *pa = p; + mmu_unmap_table(table, va, p); + return true; + } + + if ((l3_val & 0b11) != PD_TABLE) return false; + uint64_t* l3 = (uint64_t*)pt_pa_to_va(l3_val & PTE_ADDR_MASK); + + uint64_t l4_val = l3[l3_index]; + if (!(l4_val & 1)) return false; + + if (pa) *pa = l4_val & PTE_ADDR_MASK; + l3[l3_index] = 0; + if (table == (uint64_t*)kernel_ttbr0 || table == (uint64_t*)kernel_ttbr1) return true; + + bool any = false; + for (uint64_t i = 0; i < PAGE_TABLE_ENTRIES; i++) { + if (l3[i] & 1) { + any = true; + break; + } + } + + if (any) return true; + temp_free(l3, GRANULE_4KB); + l2[l2_index] = 0; + + any = false; + for (uint64_t i = 0; i < PAGE_TABLE_ENTRIES; i++) { + if (l2[i] & 1) { + any = true; + break; + } + } + + if (any) return true; + temp_free(l2, GRANULE_4KB); + l1[l1_index] = 0; + + any = false; + for (uint64_t i = 0; i < PAGE_TABLE_ENTRIES; i++) { + if (l1[i] & 1) { + any = true; + break; + } + } + + if (any) return true; + temp_free(l1, GRANULE_4KB); + table[l0_index] = 0; + + return true; +} + +bool mmu_set_access_flag(uint64_t *table, uint64_t va) { + if (!table) return false; + va &= ~(GRANULE_4KB-1); + + uint64_t l0_index = (va >> 39) & 0x1FF; + uint64_t l1_index = (va >> 30) & 0x1FF; + uint64_t l2_index = (va >> 21) & 0x1FF; + uint64_t l3_index = (va >> 12) & 0x1FF; + + uint64_t e0 = table[l0_index]; + if (!(e0 & 1)) return false; + if ((e0 & 0b11) != PD_TABLE) return false; + uint64_t *l1 = (uint64_t*)pt_pa_to_va(e0 & PTE_ADDR_MASK); + + uint64_t e1 = l1[l1_index]; + if (!(e1 & 1)) return false; + if ((e1 & 0b11) != PD_TABLE) return false; + uint64_t *l2 = (uint64_t*)pt_pa_to_va(e1 & PTE_ADDR_MASK); + + uint64_t e2 = l2[l2_index]; + if (!(e2 & 1)) return false; + + if ((e2 & 0b11) == PD_BLOCK) { + if (e2 & PTE_AF) return true; + l2[l2_index] = e2 | PTE_AF; + return true; + } + + if ((e2 & 0b11) != PD_TABLE) return false; + uint64_t *l3 = (uint64_t*)pt_pa_to_va(e2 & PTE_ADDR_MASK); + + uint64_t e3 = l3[l3_index]; + if (!(e3 & 1)) return false; + if ((e3 & 0b11) != PD_TABLE) return false; + if (e3 & PTE_AF) return true; + l3[l3_index] = e3 | PTE_AF; + return true; +} \ No newline at end of file diff --git a/kernel/memory/mmu.h b/kernel/memory/mmu.h index 0107ac43..fa7befe9 100644 --- a/kernel/memory/mmu.h +++ b/kernel/memory/mmu.h @@ -2,32 +2,66 @@ #include "types.h" +typedef struct mm_struct mm_struct; + #define GRANULE_4KB 0x1000 #define GRANULE_2MB 0x200000 +#define MMU_MAP_EXEC 0x01 + +#define PTE_ADDR_MASK 0x000FFFFFFFFFF000ULL +#define PTE_PXN (1ULL << 53) +#define PTE_UXN (1ULL << 54) +#define PTE_AF (1ULL << 10) +#define PTE_NG (1ULL << 11) +#define PTE_SH_SHIFT 8 +#define PTE_AP_SHIFT 6 +#define PTE_ATTR_SHIFT 2 + +#define PAGE_TABLE_ENTRIES 512 +#define PD_TABLE 0b11 +#define PD_BLOCK 0b01 + uint64_t* mmu_alloc(); void mmu_init(); #ifdef __cplusplus extern "C" { #endif + +#define MMU_TR_OK 0 +#define MMU_TR_ERR_PARAM 1 +#define MMU_TR_ERR_L1 2 +#define MMU_TR_ERR_L2 3 +#define MMU_TR_ERR_L3 4 +#define MMU_TR_ERR_L4 5 + void mmu_map_kernel(uintptr_t *ttbr); uintptr_t* mmu_new_ttbr(); -void register_device_memory(uint64_t va, uint64_t pa); -void register_device_memory_2mb(uint64_t va, uint64_t pa); -void register_proc_memory(uint64_t va, uint64_t pa, uint8_t attributes, uint8_t level); +void register_device_memory(kaddr_t va, paddr_t pa); +void register_device_memory_dmap(kaddr_t va); +void register_device_memory_2mb(kaddr_t va, paddr_t pa); +void register_proc_memory(uint64_t va, paddr_t pa, uint8_t attributes, uint8_t level); void mmu_map_4kb(uint64_t *table, uint64_t va, uint64_t pa, uint64_t attr_index, uint8_t mem_attributes, uint8_t level); +void mmu_unmap_table(uint64_t *table, uint64_t va, uint64_t pa); void debug_mmu_address(uint64_t va); void mmu_enable_verbose(); -void mmu_swap_ttbr(uintptr_t* ttbr); +void mmu_swap_ttbr(mm_struct *mm); +void mmu_ttbr0_disable_user(); +void mmu_ttbr0_enable_user(); +bool mmu_ttbr0_user_enabled(); +void mmu_flush_asid(uint16_t asid); +void mmu_asid_ensure(mm_struct *mm); +void mmu_asid_release(mm_struct *mm); +bool mmu_unmap_and_get_pa(uint64_t *table, uint64_t va, uint64_t *pa); +bool mmu_set_access_flag(uint64_t *table, uint64_t va); uintptr_t* mmu_default_ttbr(); void mmu_free_ttbr(uintptr_t *ttbr); -uintptr_t mmu_translate(uintptr_t va); -void mmu_map_all(uintptr_t pa); +uintptr_t mmu_translate(uint64_t *root, uintptr_t va, int *status); +void mmu_map_all(paddr_t pa); #ifdef __cplusplus } #endif -extern uintptr_t *pttbr; void mmu_unmap(uint64_t va, uint64_t pa); void mmu_init_kernel(); \ No newline at end of file diff --git a/kernel/memory/mmu_start.S b/kernel/memory/mmu_start.S deleted file mode 100644 index a5899d95..00000000 --- a/kernel/memory/mmu_start.S +++ /dev/null @@ -1,54 +0,0 @@ -#include "sysregs.h" - -.global mmu_start -mmu_start://(uint64_t *page x0) -.type mmu_start, %function - ldr x3, =MAIR_VALUE - msr mair_el1, x3 - ldr x3, =TCR_VALUE - msr tcr_el1, x3 - dsb ish - isb - - msr ttbr0_el1, x0 - msr ttbr1_el1, x0 - - mrs x0, sctlr_el1 - orr x0, x0, #0x1 - bic x0, x0, #(1 << 19) - msr sctlr_el1, x0 - isb - - ldr x1, =HIGH_VA - - ldr x0, =ksp - orr x0, x0, x1 - adrp x3, ksp - str x0, [x3] - - ldr x0, =cpec - orr x0, x0, x1 - adrp x3, cpec - str x0, [x3] - - mov x0, sp - orr x0, x0, x1 - mov sp, x0 - - mov x0, x29 - orr x0, x0, x1 - mov x29, x0 - - mov x0, x30 - orr x0, x0, x1 - mov x30, x0 - - ldr x0, [sp, #8] - orr x0, x0, x1 - str x0, [sp, #8] - - mrs x0, vbar_el1 - orr x0, x0, x1 - msr vbar_el1, x0 - - ret \ No newline at end of file diff --git a/kernel/memory/mmu_start.c b/kernel/memory/mmu_start.c new file mode 100644 index 00000000..f96360d2 --- /dev/null +++ b/kernel/memory/mmu_start.c @@ -0,0 +1,182 @@ +#include "sysregs.h" +#include "memory/mmu.h" +#include "memory/va_layout.h" +#include "memory/page_allocator.h" + +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t boot_ttbr0_l0[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t boot_ttbr1_l0[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t low_l1[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t low_l2_0[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t low_l2_1[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t low_l2_2[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t low_l2_3[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t direct_l1[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t direct_l2_0[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t direct_l2_1[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t direct_l2_2[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t direct_l2_3[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t boot_kimg_l1[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t boot_kimg_l2_0[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t boot_kimg_l2_1[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t boot_kimg_l2_2[PAGE_TABLE_ENTRIES]; +__attribute__((aligned(GRANULE_4KB), section(".boot.bss"))) +static uint64_t boot_kimg_l2_3[PAGE_TABLE_ENTRIES]; + +extern uint64_t __boot_kimg_pa_start; +extern uint64_t __boot_kimg_pa_end; + +extern void boot_vectors_el1(void); +extern void kernel_main(uint64_t board_type, uint64_t dtb_pa); +extern uint64_t boot_args[3]; + +__attribute__((section(".boot.text"), weak, noreturn)) +void boot_mmu_setup(uint64_t board_type) { + uint64_t dtb_pa = boot_args[2] ? boot_args[2] : boot_args[0]; // see dtb.c + uint64_t mmfr0 = 0; + asm volatile("mrs %0, id_aa64mmfr0_el1" : "=r"(mmfr0)); + + uint64_t ips= mmfr0 & 0xFULL; + if (ips > 6) ips = 6; + + uint64_t tcr = TCR_VALUE_BASE | (ips << 32); + + uint64_t asidbits = (mmfr0 >> 4) & 0xFULL; + if (asidbits >= 2) tcr |= (0x10ULL << 32); + + uint64_t attr_normal_exec = PTE_AF | (0b11ULL << PTE_SH_SHIFT) | ((uint64_t)MAIR_IDX_NORMAL << PTE_ATTR_SHIFT) | PTE_UXN; + uint64_t attr_normal_noexec = attr_normal_exec | PTE_PXN; + uint64_t attr_device_noexec = PTE_AF | ((uint64_t)MAIR_IDX_DEVICE << PTE_ATTR_SHIFT) | PTE_UXN | PTE_PXN; + + boot_ttbr0_l0[0] = ((uint64_t)low_l1 & PTE_ADDR_MASK) | PD_TABLE; + boot_ttbr1_l0[(HIGH_VA >> 39) & 0x1FF] = ((uint64_t)direct_l1 & PTE_ADDR_MASK) | PD_TABLE; + boot_ttbr1_l0[(KERNEL_IMAGE_VA_BASE >> 39) & 0x1FF] = ((uint64_t)boot_kimg_l1 & PTE_ADDR_MASK) | PD_TABLE; + + low_l1[0] = ((uint64_t)low_l2_0 & PTE_ADDR_MASK) | PD_TABLE; + low_l1[1] = ((uint64_t)low_l2_1 & PTE_ADDR_MASK) | PD_TABLE; + low_l1[2] = ((uint64_t)low_l2_2 & PTE_ADDR_MASK) | PD_TABLE; + low_l1[3] = ((uint64_t)low_l2_3 & PTE_ADDR_MASK) | PD_TABLE; + + direct_l1[0] = ((uint64_t)direct_l2_0 & PTE_ADDR_MASK) | PD_TABLE; + direct_l1[1] = ((uint64_t)direct_l2_1 & PTE_ADDR_MASK) | PD_TABLE; + direct_l1[2] = ((uint64_t)direct_l2_2 & PTE_ADDR_MASK) | PD_TABLE; + direct_l1[3] = ((uint64_t)direct_l2_3 & PTE_ADDR_MASK) | PD_TABLE; + + boot_kimg_l1[0] = ((uint64_t)boot_kimg_l2_0 & PTE_ADDR_MASK) | PD_TABLE; + boot_kimg_l1[1] = ((uint64_t)boot_kimg_l2_1 & PTE_ADDR_MASK) | PD_TABLE; + boot_kimg_l1[2] = ((uint64_t)boot_kimg_l2_2 & PTE_ADDR_MASK) | PD_TABLE; + boot_kimg_l1[3] = ((uint64_t)boot_kimg_l2_3 & PTE_ADDR_MASK) | PD_TABLE; + + for (uint64_t li = 0; li < PAGE_TABLE_ENTRIES; li++) { + uint64_t pa0 = li << 21; + uint64_t pa1 = (1ULL << 30) +(li << 21); + uint64_t pa2 = (2ULL << 30) +(li << 21); + uint64_t pa3 = (3ULL << 30) +(li << 21); + + low_l2_0[li] = (pa0 & PTE_ADDR_MASK) | attr_normal_exec | PD_BLOCK; + low_l2_1[li] = (pa1 & PTE_ADDR_MASK) | attr_normal_exec | PD_BLOCK; + low_l2_2[li] = (pa2 & PTE_ADDR_MASK) | attr_normal_exec | PD_BLOCK; + low_l2_3[li] = (pa3 & PTE_ADDR_MASK) | attr_normal_exec | PD_BLOCK; + + direct_l2_0[li] = (pa0 & PTE_ADDR_MASK) | attr_normal_noexec | PD_BLOCK; + direct_l2_1[li] = (pa1 & PTE_ADDR_MASK) | attr_normal_noexec | PD_BLOCK; + direct_l2_2[li] = (pa2 & PTE_ADDR_MASK) | attr_normal_noexec | PD_BLOCK; + direct_l2_3[li] = (pa3 & PTE_ADDR_MASK) | attr_normal_noexec | PD_BLOCK; + } + + for (uint64_t pa = 0x08000000ULL; pa < 0x0A000000ULL; pa += GRANULE_2MB) { + uint64_t l = (pa >> 21) & 0x1FF; + low_l2_0[l] = (pa & PTE_ADDR_MASK) | attr_device_noexec | PD_BLOCK; + direct_l2_0[l] = (pa & PTE_ADDR_MASK) | attr_device_noexec | PD_BLOCK; + } + + for (uint64_t pa = 0xFE000000ULL; pa < 0xFF000000ULL; pa += GRANULE_2MB){ + uint64_t g = (pa >> 30) & 0x1FF; + uint64_t l = (pa >> 21) & 0x1FF; + if (g == 0) { + low_l2_0[l] = (pa & PTE_ADDR_MASK) | attr_device_noexec | PD_BLOCK; + direct_l2_0[l] = (pa & PTE_ADDR_MASK) | attr_device_noexec | PD_BLOCK; + } + if (g == 1) { + low_l2_1[l] = (pa & PTE_ADDR_MASK) | attr_device_noexec | PD_BLOCK; + direct_l2_1[l] = (pa & PTE_ADDR_MASK) | attr_device_noexec | PD_BLOCK; + } + if (g == 2) { + low_l2_2[l] = (pa & PTE_ADDR_MASK) | attr_device_noexec | PD_BLOCK; + direct_l2_2[l] = (pa & PTE_ADDR_MASK) | attr_device_noexec | PD_BLOCK; + } + if (g == 3) { + low_l2_3[l] = (pa & PTE_ADDR_MASK) | attr_device_noexec | PD_BLOCK; + direct_l2_3[l] = (pa & PTE_ADDR_MASK) | attr_device_noexec | PD_BLOCK; + } + } + + low_l1[64] = (0x1000000000ULL & PTE_ADDR_MASK) | attr_device_noexec | PD_BLOCK; + low_l1[65] = (0x1040000000ULL & PTE_ADDR_MASK) | attr_device_noexec | PD_BLOCK; + low_l1[124] = (0x1F00000000ULL & PTE_ADDR_MASK) | attr_device_noexec | PD_BLOCK; + + uint64_t kstart = __boot_kimg_pa_start; + uint64_t kend = __boot_kimg_pa_end; + uint64_t ks = kstart & ~(GRANULE_2MB - 1ULL); + uint64_t ke = (kend + (GRANULE_2MB - 1ULL)) & ~(GRANULE_2MB - 1ULL); + + for (uint64_t pa = ks; pa < ke; pa += GRANULE_2MB){ + uint64_t g = (pa >> 30) & 0x1FF; + uint64_t l = (pa >> 21) & 0x1FF; + + if (g == 0) boot_kimg_l2_0[l] = (pa & PTE_ADDR_MASK) | attr_normal_exec | PD_BLOCK; + if (g == 1) boot_kimg_l2_1[l] = (pa & PTE_ADDR_MASK) | attr_normal_exec | PD_BLOCK; + if (g == 2) boot_kimg_l2_2[l] = (pa & PTE_ADDR_MASK) | attr_normal_exec | PD_BLOCK; + if (g == 3) boot_kimg_l2_3[l] = (pa & PTE_ADDR_MASK) | attr_normal_exec | PD_BLOCK; + } + + asm volatile("msr mair_el1, %0" :: "r"((uint64_t)MAIR_VALUE)); + asm volatile("msr tcr_el1, %0" :: "r"(tcr)); + asm volatile("dsb ish"); + asm volatile("isb"); + + asm volatile("msr ttbr0_el1, %0" :: "r"((uint64_t)boot_ttbr0_l0 & PTE_ADDR_MASK)); + asm volatile("msr ttbr1_el1, %0" :: "r"((uint64_t)boot_ttbr1_l0 & PTE_ADDR_MASK)); + + uint64_t sctlr = 0; + asm volatile("mrs %0, sctlr_el1" : "=r"(sctlr)); + sctlr |= 1ULL; + sctlr &= ~(1ULL << 19); + asm volatile("msr sctlr_el1, %0" :: "r"(sctlr)); + asm volatile("dsb ish"); + asm volatile("isb"); + + asm volatile( + "ldr x0, =boot_vectors_el1\n" + "msr vbar_el1, x0\n" + "isb\n" + ::: "x0", "memory" + ); + + asm volatile( + "ldr x2, =ksp\n" + "mov sp, x2\n" + "mov x0, %0\n" + "mov x1, %1\n" + "ldr x2, =kernel_main\n" + "blr x2\n" + "1: wfe\n" + "b 1b\n" + :: "r"(board_type),"r"(dtb_pa) : "x0", "x1", "x2", "memory" + ); +} diff --git a/kernel/memory/page_allocator.c b/kernel/memory/page_allocator.c index 3a8f7f89..a5f8387f 100644 --- a/kernel/memory/page_allocator.c +++ b/kernel/memory/page_allocator.c @@ -6,6 +6,8 @@ #include "math/math.h" #include "console/kio.h" #include "sysregs.h" +#include "memory/addr.h" +#include "exceptions/exception_handler.h" #define PD_TABLE 0b11 #define PD_BLOCK 0b01 @@ -15,14 +17,73 @@ #define PAGE_TABLE_ENTRIES 65536 +#define LOW_ADDR_WARN 0x100000ULL + +#define ALLOC_TAG_MAGIC 0xCCDEC00ED00DAA0EULL +#define ALLOC_TAG_MAGIC_INV (~ALLOC_TAG_MAGIC) +#define ALLOC_TAG_SIZE 64 +#define ALLOC_KIND_SMALL 1 + +typedef struct { + uint64_t magic; + uint64_t magic_inv; + uint64_t base_phys; + uint64_t user_phys; + uint64_t owner_phys; + uint32_t alloc_size; + uint32_t user_size; + uint16_t alignment; + uint8_t kind; + uint8_t level; + uint8_t attributes; + uint8_t reserved0; + uint16_t reserved1; + uint32_t checksum; + uint32_t checksum_inv; +} alloc_tag; + +typedef struct { + uint64_t phys_base; + uint64_t size; + uint64_t owner_phys; +} big_alloc_entry; + +#define BIG_ALLOC_ENTRIES ((PAGE_SIZE - 16) / sizeof(big_alloc_entry)) + +typedef struct big_alloc_page { + struct big_alloc_page* next; + uint32_t count; + uint32_t pad; + big_alloc_entry entries[BIG_ALLOC_ENTRIES]; +} big_alloc_page; + + uintptr_t *mem_bitmap; +static big_alloc_page* big_alloc_meta = 0; + +static uint64_t alloc_min_page = 0; +static uint64_t alloc_max_page = 0; +static uint64_t alloc_hint_page = 0; +static uint64_t bitmap_page_count = 0; static bool page_alloc_verbose = false; +extern uintptr_t heap_end; +static void page_alloc_init(); + void page_alloc_enable_verbose(){ page_alloc_verbose = true; } +void page_alloc_enable_high_va(){ + uint64_t sctlr = 0; + asm volatile("mrs %0, sctlr_el1" : "=r"(sctlr)); + if ((sctlr & 1) == 0) return; + if (!mem_bitmap) return; + if (((uintptr_t)mem_bitmap & HIGH_VA) == HIGH_VA) return; + mem_bitmap = (uintptr_t*)PHYS_TO_VIRT((uintptr_t)mem_bitmap); +} + #define kprintfv(fmt, ...) \ ({ \ if (page_alloc_verbose){\ @@ -34,68 +95,129 @@ uint64_t count_pages(uint64_t i1,uint64_t i2){ return (i1/i2) + (i1 % i2 > 0); } +static inline uint64_t lowmask64(uint64_t bits) { + if (!bits) return 0; + if (bits >= 64) return UINT64_MAX; + return (1ull << bits) - 1ull; +} + void pfree(void* ptr, uint64_t size) { - int pages = count_pages(size,PAGE_SIZE); + if (!ptr || !size) return; + if (!alloc_max_page) page_alloc_init(); + page_alloc_enable_high_va(); + if (!mem_bitmap || !alloc_max_page) panic("pfree init failed", (uintptr_t)ptr); + + uint64_t pages = count_pages(size,PAGE_SIZE); uint64_t addr = VIRT_TO_PHYS((uint64_t)ptr); addr /= PAGE_SIZE; - for (int i = 0; i < pages; i++){ + if (addr < alloc_min_page || addr + pages > alloc_max_page) panic("pfree out of range", (uintptr_t)ptr); + + for (uint64_t i = 0; i < pages; i++){ uint64_t index = addr + i; uint64_t table_index = index/64; uint64_t table_offset = index % 64; mem_bitmap[table_index] &= ~(1ULL << table_offset); - mmu_unmap(index * PAGE_SIZE, index * PAGE_SIZE); } -} -void free_page_list(page_index *index){ - if (index->header.next) free_page_list(index->header.next); - for (size_t i = 0; i < index->header.size; i++) - pfree(index->ptrs[i].ptr, index->ptrs[i].size); + if (addr < alloc_hint_page) alloc_hint_page = addr; + if (alloc_hint_page < alloc_min_page) alloc_hint_page = alloc_min_page; } void free_managed_page(void* ptr){ + if (!ptr) return; + + uintptr_t owner_phys = (uintptr_t)ptr; + if ((owner_phys & HIGH_VA) == HIGH_VA) owner_phys = VIRT_TO_PHYS(owner_phys); + owner_phys &= ~0xFFFULL; + + big_alloc_page* mp = big_alloc_meta; + while (mp){ + for (uint32_t i = 0; i < mp->count;){ + if (mp->entries[i].owner_phys != owner_phys){ + i++; + continue; + } + + uint64_t phys_base = mp->entries[i].phys_base; + uint64_t size = mp->entries[i].size; + + mp->count--; + mp->entries[i] = mp->entries[mp->count]; + + pfree(PHYS_TO_VIRT_P((void*)phys_base), size); + } + mp = mp->next; + } + mem_page *info = (mem_page*)ptr; if (info->next) free_managed_page(info->next); - if (info->page_alloc) - free_page_list(info->page_alloc); pfree((void*)ptr, PAGE_SIZE); } -uint64_t start; -uint64_t end; +static void page_alloc_init(){ + uint64_t ram_start = get_user_ram_start(); + uint64_t ram_end = get_user_ram_end(); + uint64_t start_page = ram_start / PAGE_SIZE; + uint64_t end_page = ram_end / PAGE_SIZE; + + uint64_t words = (end_page + 63) /64; + uint64_t bytes = words * sizeof(uint64_t); + bitmap_page_count = count_pages(bytes, PAGE_SIZE); + + uint64_t sctlr = 0; + asm volatile("mrs %0, sctlr_el1" : "=r"(sctlr)); + if ((sctlr & 1) != 0) mem_bitmap = (uintptr_t*)PHYS_TO_VIRT(ram_start); + else mem_bitmap = (uintptr_t*)ram_start; + memset(mem_bitmap, 0, bitmap_page_count * PAGE_SIZE); + + if (end_page & 63) { + uint64_t tail = end_page & 63; + mem_bitmap[words - 1] |= ~lowmask64(tail); + } + + alloc_min_page = start_page + bitmap_page_count; + alloc_max_page = end_page; + alloc_hint_page = alloc_min_page; + + heap_end = alloc_min_page * PAGE_SIZE; + mark_used(ram_start, bitmap_page_count); +} void setup_page(uintptr_t address, uint8_t attributes){ - mem_page* new_info = (mem_page*)address; + uint64_t sctlr = 0; + asm volatile("mrs %0, sctlr_el1" : "=r"(sctlr)); + mem_page* new_info = (mem_page*)(((sctlr & 1) != 0) ? PHYS_TO_VIRT(address) : address); memset(new_info, 0, sizeof(mem_page)); new_info->next_free_mem_ptr = address + sizeof(mem_page); new_info->attributes = attributes; } -extern uintptr_t heap_end; - -void* palloc_inner(uint64_t size, uint8_t level, uint8_t attributes, bool full, bool map) { - if (!start || !end) { - start = count_pages(get_user_ram_start(),PAGE_SIZE); - end = count_pages(get_user_ram_end(),PAGE_SIZE); - mem_bitmap = (uintptr_t*)(start * PAGE_SIZE); - start += count_pages(65536*8,PAGE_SIZE); - memset(mem_bitmap, 0, 65536*8); - heap_end = start*PAGE_SIZE; - } +paddr_t palloc_inner(uint64_t size, uint8_t level, uint8_t attributes, bool full, bool map) { + if (!alloc_max_page) page_alloc_init(); + page_alloc_enable_high_va(); uint64_t page_count = count_pages(size,PAGE_SIZE); + uint64_t reg_min = alloc_min_page / 64; + uint64_t reg_end = (alloc_max_page + 63) / 64; + uint64_t reg_hint = alloc_hint_page / 64; if (page_count > 64){ kprintfv("[page_alloc] Large allocation > 64p"); uint64_t reg_count = page_count/64; uint8_t fractional = page_count % 64; reg_count += fractional > 0; - - for (uint64_t i = start/64; i < end/64; i++) { + uint64_t align_regs = 1; + if (size >= GRANULE_2MB && (size & (GRANULE_2MB - 1)) == 0) align_regs = GRANULE_2MB / (PAGE_SIZE * 64); + + for (int pass = 0; pass < 2; pass++) { + uint64_t i0 = pass == 0 ? reg_hint : reg_min; + uint64_t i1 = pass == 0 ? reg_end : reg_hint; + for (uint64_t i = i0; i + reg_count <= i1; i++) { + if (align_regs > 1 && (i % align_regs) != 0) continue; bool found = true; for (uint64_t j = 0; j < reg_count; j++){ if (fractional && j == reg_count-1) - found &= (mem_bitmap[i + j] & ((1ULL << (fractional + 1)) - 1)) == 0; + found &= (mem_bitmap[i + j] & lowmask64(fractional)) == 0; else found &= mem_bitmap[i + j] == 0; @@ -104,56 +226,69 @@ void* palloc_inner(uint64_t size, uint8_t level, uint8_t attributes, bool full, if (found){ for (uint64_t j = 0; j < reg_count; j++){ if (fractional && j == reg_count-1) - mem_bitmap[i+j] |= ((1ULL << (fractional + 1)) - 1); + mem_bitmap[i+j] |= lowmask64(fractional); else mem_bitmap[i+j] = UINT64_MAX; } - - start = (i + (reg_count - (fractional > 0))) * 64; + + alloc_hint_page = (i * 64) + page_count; + if (alloc_hint_page < alloc_min_page) alloc_hint_page = alloc_min_page; + mem_page* prev_page = 0; for (uint32_t p = 0; p < page_count; p++){ uintptr_t address = ((i * 64) + p) * PAGE_SIZE; if (map){ if ((attributes & MEM_DEV) != 0 && level == MEM_PRIV_KERNEL) register_device_memory(address, address); - else + else if (level != MEM_PRIV_USER) register_proc_memory(address, address, attributes, level); + if (!full){ + setup_page(address, attributes); + + mem_page* curr = (mem_page*)PHYS_TO_VIRT(address); + if (prev_page) prev_page->next = curr; + prev_page = curr; + + memset((void*)PHYS_TO_VIRT(address +sizeof(mem_page)), 0, PAGE_SIZE - sizeof(mem_page)); + } else { + memset((void*)PHYS_TO_VIRT(address), 0, PAGE_SIZE); + } } } kprintfv("[page_alloc] Final address %x", (i * 64 * PAGE_SIZE)); - void* addr = (void*)(i * 64 * PAGE_SIZE); - if (map) memset(PHYS_TO_VIRT_P(addr), 0, size); - return addr; + return (paddr_t)(i * 64 * PAGE_SIZE); } } + } + return 0; } - bool skipped_regs = false; + for (int pass = 0; pass < 2; pass++) { + uint64_t i0 = pass == 0 ? reg_hint : reg_min; + uint64_t i1 = pass == 0 ? reg_end : reg_hint; + + for (uint64_t i = i0; i < i1; i++) { + if (mem_bitmap[i] == UINT64_MAX) continue; - for (uint64_t i = start/64; i < end/64; i++) { - if (mem_bitmap[i] != UINT64_MAX) { - kprintfv("Normal allocation"); uint64_t inv = ~mem_bitmap[i]; uint64_t bit = __builtin_ctzll(inv); if (bit > (64 - page_count)){ - skipped_regs = true; continue; } - do { + while (bit < 64) { bool found = true; - for (uint64_t b = bit; b < (uint64_t)min(64,bit + (page_count - 1)); b++){ - if (((mem_bitmap[i] >> b) & 1)){ - bit += page_count; + for (uint64_t b = bit; b < bit + page_count; b++){ + if ((mem_bitmap[i] >> b) & 1ull){ + bit = b + 1; found = false; + break; } } if (found) break; - } while (bit < 64); - if (bit == 64){ - skipped_regs = true; - continue; } - + if (bit >= 64) continue; uintptr_t first_address = 0; + mem_page* prev_page = 0; + for (uint64_t j = 0; j < page_count; j++){ mem_bitmap[i] |= (1ULL << (bit + j)); uint64_t page_index = (i * 64) + (bit + j); @@ -163,22 +298,28 @@ void* palloc_inner(uint64_t size, uint8_t level, uint8_t attributes, bool full, if (map){ if ((attributes & MEM_DEV) != 0 && level == MEM_PRIV_KERNEL) register_device_memory(address, address); - else + else if (level != MEM_PRIV_USER) register_proc_memory(address, address, attributes, level); - } - if (!full && map) { - setup_page(address, attributes); + if (!full) { + setup_page(address, attributes); + mem_page* curr = (mem_page*)PHYS_TO_VIRT(address); + if (prev_page) prev_page->next = curr; + prev_page = curr; + + memset((void*)PHYS_TO_VIRT(address + sizeof(mem_page)), 0, PAGE_SIZE - sizeof(mem_page)); + } else { + memset((void*)PHYS_TO_VIRT(address), 0, PAGE_SIZE); + } } } + alloc_hint_page = (first_address / PAGE_SIZE) + page_count; + if (alloc_hint_page < alloc_min_page) alloc_hint_page = alloc_min_page; + kprintfv("[page_alloc] Final address %x", first_address); - if (map){ - size_t extra = full ? 0 : sizeof(mem_page); - memset((void*)PHYS_TO_VIRT((first_address + extra)),0,size - extra); - } - return (void*)first_address; - } else if (!skipped_regs) start = (i + 1) * 64; + return (paddr_t)first_address; + } } uart_puts("[page_alloc error] Could not allocate"); @@ -186,14 +327,16 @@ void* palloc_inner(uint64_t size, uint8_t level, uint8_t attributes, bool full, } void* palloc(uint64_t size, uint8_t level, uint8_t attributes, bool full){ - void* phys = palloc_inner(size, level, attributes, full, true); + paddr_t phys = palloc_inner(size, level, attributes, full, true); if(!phys) return 0; - return PHYS_TO_VIRT_P(phys); + return (void*)dmap_pa_to_kva(phys); } bool page_used(uintptr_t ptr){ - uint64_t addr = (uint64_t)ptr; - addr /= PAGE_SIZE; + page_alloc_enable_high_va(); + if (!mem_bitmap || !alloc_max_page) return false; + uint64_t addr = VIRT_TO_PHYS((uint64_t)ptr) / PAGE_SIZE; + if (addr >= alloc_max_page) return false; uint64_t table_index = addr/64; uint64_t table_offset = addr % 64; return (mem_bitmap[table_index] >> table_offset) & 1; @@ -201,16 +344,16 @@ bool page_used(uintptr_t ptr){ void mark_used(uintptr_t address, size_t pages) { + page_alloc_enable_high_va(); + if (!mem_bitmap) return; + address = VIRT_TO_PHYS(address); if ((address & (PAGE_SIZE - 1)) != 0) { kprintf("[mark_used error] address %x not aligned", address); return; } if (pages == 0) return; - uint64_t start = count_pages(get_user_ram_start(),PAGE_SIZE); - - uint64_t page_index = (address / (PAGE_SIZE * 64)) - (start/64); - + uint64_t page_index = address / PAGE_SIZE; for (size_t j = 0; j < pages; j++) { uint64_t idx = page_index + j; uint64_t i = idx / 64; @@ -219,94 +362,191 @@ void mark_used(uintptr_t address, size_t pages) mem_bitmap[i] |= (1ULL << bit); } } - void* kalloc_inner(void *page, size_t size, uint16_t alignment, uint8_t level, uintptr_t page_va, uintptr_t *next_va, uintptr_t *ttbr){ if (!page) return 0; + if (!size) return 0; + if (!alignment || (alignment & (alignment - 1))) { + kprintfv("[kalloc] bad alignment %x", alignment); + return 0; + } + + size_t req_size = size; size = (size + alignment - 1) & ~(alignment - 1); - kprintfv("[in_page_alloc] Requested size: %x", size); + if ((uintptr_t)page < LOW_ADDR_WARN) kprintfv("[kalloc an] low page=%llx size=%llx align=%x", (uint64_t)(uintptr_t)page, (uint64_t)size, (uint32_t)alignment); + + if (size & 0xFULL) size = (size + 15) & ~0xFULL; mem_page *info = (mem_page*)PHYS_TO_VIRT_P(page); + + uintptr_t owner_phys = (uintptr_t)page; + if ((owner_phys & HIGH_VA) == HIGH_VA) owner_phys = VIRT_TO_PHYS(owner_phys); + owner_phys &= ~0xFFFULL; + if (!info->next_free_mem_ptr){ uintptr_t page_phys = (uintptr_t)page; if ((page_phys & HIGH_VA) == HIGH_VA) page_phys = VIRT_TO_PHYS(page_phys); - setup_page(page_phys, info->attributes); - + page_phys &= ~0xFFFULL; + setup_page(page_phys, info->attributes);//b info = (mem_page*)PHYS_TO_VIRT_P((void*)page_phys); } - - if (size >= PAGE_SIZE){ - void* ptr = palloc(size, level, info->attributes, true); - page_index *index = info->page_alloc; - if (!index){ - info->page_alloc = palloc(PAGE_SIZE, level, info->attributes, true); - index = info->page_alloc; + + size_t small_need = ALLOC_TAG_SIZE + size + (alignment - 1); + if (small_need & 0xFULL) small_need = (small_need + 15) & ~0xFULL; + + if (size >= PAGE_SIZE || alignment >= PAGE_SIZE || small_need >= PAGE_SIZE) { + uint64_t alloc_size = size; + if (alloc_size & (alignment - 1)) alloc_size = (alloc_size + alignment - 1) & ~(alignment - 1); + alloc_size = (alloc_size + PAGE_SIZE - 1) & ~((uint64_t)PAGE_SIZE - 1); + + void* ptr = palloc(alloc_size, level, info->attributes, true);//b + if (!ptr) return 0; + + uintptr_t phys_base = VIRT_TO_PHYS((uintptr_t)ptr); + + big_alloc_page* mp = big_alloc_meta; + while (mp && mp->count >= BIG_ALLOC_ENTRIES) + mp = mp->next; + + if (!mp) { + paddr_t meta_phys =palloc_inner(PAGE_SIZE, MEM_PRIV_KERNEL, MEM_RW, true, true); + if (!meta_phys) panic("kalloc no metadata page", alloc_size); + mp = (big_alloc_page*)dmap_pa_to_kva(meta_phys); + memset(mp, 0, PAGE_SIZE); + mp->next = big_alloc_meta; + big_alloc_meta = mp; } - register_allocation(index, ptr, size); + + mp->entries[mp->count].phys_base = phys_base; + mp->entries[mp->count].size = alloc_size; + mp->entries[mp->count].owner_phys = owner_phys; + mp->count++; + if (page_va && next_va && ttbr){ uintptr_t va = *next_va; - for (uintptr_t i = (uintptr_t)ptr; i < (uintptr_t)ptr + size; i+= GRANULE_4KB){ - mmu_map_4kb(ttbr, *next_va, (uintptr_t)i, (info->attributes & MEM_DEV) ? MAIR_IDX_DEVICE : MAIR_IDX_NORMAL, info->attributes, level); + for (uint64_t i = 0; i < alloc_size; i+= GRANULE_4KB){ + mmu_map_4kb((uint64_t*)ttbr, (uint64_t)*next_va, (paddr_t)(phys_base + i), (info->attributes & MEM_DEV) ? MAIR_IDX_DEVICE : MAIR_IDX_NORMAL, info->attributes, level); *next_va += PAGE_SIZE; } return (void*)va; } + memset((void*)PHYS_TO_VIRT(phys_base), 0, alloc_size); return ptr; } FreeBlock** curr = PHYS_TO_VIRT_P(&info->free_list); + if (info->free_list && (uintptr_t)info->free_list < LOW_ADDR_WARN) + kprintfv("[kalloc an] free_list low head=%llx page=%llx", (uint64_t)(uintptr_t)info->free_list, (uint64_t)(uintptr_t)page); + FreeBlock *cblock = PHYS_TO_VIRT_P(*curr); while (*curr && ((uintptr_t)*curr & 0xFFFFFFFF) != 0xDEADBEEF && (uintptr_t)cblock != 0xDEADBEEF && (uintptr_t)cblock != 0xDEADBEEFDEADBEEF) { - if (cblock->size >= size) { - kprintfv("[in_page_alloc] Reusing free block at %x",(uintptr_t)*curr); - - uint64_t result = (uint64_t)cblock; - //*curr = VIRT_TO_PHYS_P(cblock->next); - *curr = cblock->next; - memset((void*)PHYS_TO_VIRT(result), 0, size); - info->size += size; - if (page_va){ - return (void*)(page_va | (result & 0xFFF)); - } - return (void*)result; + uintptr_t base_phys = (uintptr_t)*curr; + uint64_t bsz = cblock->size; + + if (bsz >= small_need) { + uintptr_t user_phys = base_phys + ALLOC_TAG_SIZE; + if (user_phys & (alignment - 1)) user_phys = (user_phys + alignment - 1) & ~(uintptr_t)(alignment - 1); + uintptr_t tag_phys = user_phys - ALLOC_TAG_SIZE; + + if (user_phys + size <= base_phys + bsz) { + *curr = cblock->next; + + alloc_tag* tag = (alloc_tag*)PHYS_TO_VIRT(tag_phys); + tag->magic = ALLOC_TAG_MAGIC; + tag->magic_inv = ALLOC_TAG_MAGIC_INV; + tag->base_phys = base_phys; + tag->user_phys = user_phys; + tag->owner_phys = owner_phys; + tag->alloc_size = (uint32_t)bsz; + tag->user_size = (uint32_t)req_size; + tag->alignment = alignment; + tag->kind = ALLOC_KIND_SMALL; + tag->level = level; + tag->attributes = info->attributes; + tag->reserved0 = 0; + tag->reserved1 = 0; + + uint64_t mix = tag->base_phys ^ tag->user_phys ^ tag->owner_phys ^ ((uint64_t)tag->alloc_size << 32) ^ tag->user_size; + mix ^= ((uint64_t)tag->alignment << 48) ^ ((uint64_t)tag->kind << 40) ^ ((uint64_t)tag->level << 32) ^ ((uint64_t)tag->attributes << 24); + mix ^= ALLOC_TAG_MAGIC; //token + uint32_t c = (uint32_t)(mix ^ (mix >> 32)); + tag->checksum = c; + tag->checksum_inv = ~c; + + memset((void*)PHYS_TO_VIRT(user_phys), 0, size); + info->size += bsz; + if (page_va) return(void*)(page_va | (user_phys & 0xFFF)); + return (void*)user_phys; + } } - kprintfv("-> %x",(uintptr_t)&cblock->next); - curr = &(cblock)->next; + + curr = &cblock->next; cblock = PHYS_TO_VIRT_P(*curr); } + uintptr_t page_phys = (uintptr_t)page; + if ((page_phys & HIGH_VA) == HIGH_VA) page_phys = VIRT_TO_PHYS(page_phys); + page_phys &= ~0xFFFULL; + + uintptr_t base_phys = info->next_free_mem_ptr; + if (base_phys & 0xFULL) base_phys = (base_phys + 15) & ~0xFULL; kprintfv("[in_page_alloc] Current next pointer %llx",info->next_free_mem_ptr); - info->next_free_mem_ptr = (info->next_free_mem_ptr + alignment - 1) & ~(alignment - 1); + uintptr_t user_phys = base_phys + ALLOC_TAG_SIZE; + if (user_phys & (alignment - 1)) user_phys = (user_phys + alignment - 1) & ~(uintptr_t)(alignment - 1); + uintptr_t tag_phys = user_phys - ALLOC_TAG_SIZE; - kprintfv("[in_page_alloc] Aligned next pointer %llx",info->next_free_mem_ptr); + kprintfv("[in_page_alloc] Aligned next pointer %llx", base_phys); - if (info->next_free_mem_ptr + size > ((VIRT_TO_PHYS((uintptr_t)page)) + PAGE_SIZE)) { - uintptr_t next_page_va = page_va + PAGE_SIZE; + if (base_phys + small_need > page_phys + PAGE_SIZE) { + uintptr_t next_page_va = page_va ? (page_va + PAGE_SIZE) : 0; + if (next_va) next_page_va = *next_va; if (!info->next){ info->next = palloc(PAGE_SIZE, level, info->attributes, false); if (next_va) next_page_va = *next_va; if (page_va && next_va && ttbr){ - mmu_map_4kb(ttbr, *next_va, (uintptr_t)info->next, (info->attributes & MEM_DEV) ? MAIR_IDX_DEVICE : MAIR_IDX_NORMAL, info->attributes, level); + uintptr_t phys_next = VIRT_TO_PHYS((uintptr_t)info->next); + register_proc_memory((uintptr_t)*next_va, (paddr_t)phys_next, info->attributes, level); *next_va += PAGE_SIZE; } kprintfv("[in_page_alloc] Page %llx points to new page %llx",page,info->next); } - kprintfv("[in_page_alloc] Page full. Moving to %x",(uintptr_t)info->next); - return kalloc_inner(info->next, size, alignment, level, page_va ? next_page_va : 0, next_va, ttbr); + kprintfv("[in_page_alloc] Page full. Moving to %x", (uintptr_t)info->next); + return kalloc_inner(info->next, req_size, alignment, level, next_page_va, next_va, ttbr); } - uint64_t result = info->next_free_mem_ptr; - info->next_free_mem_ptr += size; - - kprintfv("[in_page_alloc] Allocated address %x",result); + info->next_free_mem_ptr = base_phys + small_need; + + alloc_tag* tag = (alloc_tag*)PHYS_TO_VIRT(tag_phys); + tag->magic = ALLOC_TAG_MAGIC; + tag->magic_inv = ALLOC_TAG_MAGIC_INV; + tag->base_phys = base_phys; + tag->user_phys = user_phys; + tag->owner_phys = owner_phys; + tag->alloc_size = (uint32_t)small_need; + tag->user_size = (uint32_t)req_size; + tag->alignment = alignment; + tag->kind = ALLOC_KIND_SMALL; + tag->level = level; + tag->attributes = info->attributes; + tag->reserved0 = 0; + tag->reserved1 = 0; + + uint64_t mix = tag->base_phys ^ tag->user_phys ^ tag->owner_phys ^ ((uint64_t)tag->alloc_size << 32) ^ tag->user_size; + mix ^= ((uint64_t)tag->alignment << 48) ^ ((uint64_t)tag->kind << 40) ^ ((uint64_t)tag->level << 32) ^ ((uint64_t)tag->attributes << 24); + mix ^= ALLOC_TAG_MAGIC; + uint32_t c = (uint32_t)(mix ^ (mix >> 32)); + tag->checksum = c; + tag->checksum_inv = ~c; + + memset((void*)PHYS_TO_VIRT(user_phys), 0, size); + info->size += small_need; + kprintfv("[in_page_alloc] Allocated address %x",user_phys); - memset((void*)PHYS_TO_VIRT(result), 0, size); - info->size += size; if (page_va){ - return (void*)(page_va | (result & 0xFFF)); + return (void*)(page_va | (user_phys & 0xFFF)); } - return (void*)result; + return (void*)user_phys; } void* make_page_index(){ @@ -346,6 +586,8 @@ void free_registered(page_index *index, void *ptr){ for (u64 i = 0; i < ind->header.size; i++){ if (ind->ptrs[i].ptr == ptr){ pfree(ind->ptrs[i].ptr, ind->ptrs[i].size); + ind->ptrs[i] = ind->ptrs[ind->header.size - 1]; + ind->header.size--; return; } } @@ -368,28 +610,117 @@ void release_page_index(page_index *index){ void* kalloc(void *page, size_t size, uint16_t alignment, uint8_t level){ void* ptr = kalloc_inner(page, size, alignment, level, 0, 0, 0); - if (level == MEM_PRIV_KERNEL) ptr = PHYS_TO_VIRT_P(ptr); + if (level == MEM_PRIV_KERNEL && ptr) ptr = PHYS_TO_VIRT_P(ptr); return ptr; } void kfree(void* ptr, size_t size) { - if(!ptr || size == 0) return; + if(!ptr) return; kprintfv("[page_alloc_free] Freeing block at %x size %x",(uintptr_t)ptr, size); - if(size & 0xF) size = (size + 15) & ~0xFULL; + uintptr_t va = (uintptr_t)ptr; + uintptr_t phys = 0; + + if((va & HIGH_VA) == HIGH_VA){ + phys = VIRT_TO_PHYS(va); + } else { + int tr = 0; + phys = mmu_translate(0, va, &tr); + if(tr){ + kprintf("[kfree] unmapped ptr=%llx size=%llx", (uint64_t)va, (uint64_t)size); + panic("kfree unmapped pointer", va); + } + } + + uintptr_t phys_base = phys& ~0xFFFULL; + uint64_t page_off = va & 0xFFFULL; - memset32((void*)ptr,0,size); + bool tag_ok = false; + alloc_tag* tag = 0; - mem_page *page = (mem_page *)(((uintptr_t)ptr) & ~0xFFFULL); - uintptr_t phys_page = mmu_translate((uintptr_t)page); - uintptr_t off = (uintptr_t)ptr & 0xFFFULL; - uintptr_t block_phys = phys_page | off; + if(page_off >= ALLOC_TAG_SIZE) { + uintptr_t phys_tag = 0; + if((va & HIGH_VA) == HIGH_VA) { + phys_tag = VIRT_TO_PHYS(va - ALLOC_TAG_SIZE); + } else { + int tr = 0; + phys_tag = mmu_translate(0, va - ALLOC_TAG_SIZE, &tr); + if(tr) phys_tag = 0; + } + + if(phys_tag) { + tag = (alloc_tag*)PHYS_TO_VIRT(phys_tag); + if(tag->magic == ALLOC_TAG_MAGIC && tag->magic_inv == ALLOC_TAG_MAGIC_INV && tag->user_phys == phys){ + uint64_t mix = tag->base_phys ^ tag->user_phys ^ tag->owner_phys ^ ((uint64_t)tag->alloc_size << 32) ^ tag->user_size; + mix ^= ((uint64_t)tag->alignment << 48) ^ ((uint64_t)tag->kind << 40) ^ ((uint64_t)tag->level << 32) ^ ((uint64_t)tag->attributes << 24); + mix ^= ALLOC_TAG_MAGIC; //token + uint32_t c = (uint32_t)(mix ^ (mix >> 32)); + if(tag->checksum == c && tag->checksum_inv == ~c) tag_ok = true; + } + } + } + + if(tag_ok) { + if(tag->kind != ALLOC_KIND_SMALL) { + kprintf("[kfree] bad tag kind ptr=%llx phys=%llx kind=%u size=%llx", (uint64_t)va, (uint64_t)phys, (unsigned)tag->kind, (uint64_t)size); + panic("kfree bad tag kind", va); + } + + uintptr_t base_phys = (uintptr_t)tag->base_phys; + uint64_t alloc_size = tag->alloc_size; + + if(!alloc_size || alloc_size >= PAGE_SIZE) { + kprintf("[kfree] bad small size ptr=%llx phys=%llx base=%llx alloc=%llx", (uint64_t)va, (uint64_t)phys, (uint64_t)base_phys, alloc_size); + panic("kfree bad small alloc size", base_phys); + } + + uintptr_t page_phys = base_phys & ~0xFFFULL; + mem_page *page = (mem_page *)PHYS_TO_VIRT(page_phys); + + memset32((void*)PHYS_TO_VIRT(base_phys),0xDEADBEEF,alloc_size); + FreeBlock* block = (FreeBlock*)PHYS_TO_VIRT( base_phys); + block->size = alloc_size; + block->next = page->free_list; + page->free_list = (FreeBlock*)base_phys; + + if(page->size >= alloc_size) page->size -= alloc_size; + else page->size = 0; + + return; + } + + if(page_off != 0) { + kprintf("[kfree] untracked ptr=%llx phys=%llx size=%llx off=%llx", (uint64_t)va, (uint64_t)phys, (uint64_t)size, page_off); + panic("kfree untracked pointer", va); + } + + big_alloc_page* mp = big_alloc_meta; + while(mp) { + for(uint32_t i = 0; i < mp->count; i++) { + if(mp->entries[i].phys_base != phys_base) continue; + + if(mp->entries[i].phys_base != phys) { + kprintf("[kfree] non base big ptr=%llx phys=%llx base=%llx", (uint64_t)va, (uint64_t)phys, (uint64_t)phys_base); + panic("kfree non base big pointer", va); + } + + uint64_t big_size = mp->entries[i].size; + mp->count--; + mp->entries[i] = mp->entries[mp->count]; + + if(!mem_bitmap) { + kprintf("[kfree] bitmap not init ptr=%llx phys=%llx", (uint64_t)va, (uint64_t)phys); + panic("kfree bitmap not init", va); + } + + pfree(PHYS_TO_VIRT_P((void*)phys_base), big_size); + return; + } + mp = mp->next; + } - FreeBlock* block = (FreeBlock*)PHYS_TO_VIRT(block_phys); - block->size = size; - block->next = page->free_list; - page->free_list = (FreeBlock*)block_phys; - page->size -= size; + kprintf("[kfree] page pointer not tracked ptr=%llx phys=%llx size=%llx", (uint64_t)va, (uint64_t)phys, (uint64_t)size); + panic("kfree page pointer not tracked (use pfree)", phys); } void free_sizedptr(sizedptr ptr){ diff --git a/kernel/memory/page_allocator.h b/kernel/memory/page_allocator.h index f81d85f8..0c627c2d 100644 --- a/kernel/memory/page_allocator.h +++ b/kernel/memory/page_allocator.h @@ -18,12 +18,13 @@ extern "C" { #endif void page_alloc_enable_verbose(); +void page_alloc_enable_high_va(); void* make_page_index(); void register_allocation(page_index *index, void* ptr, size_t size);//DEADLINE: 01/03/2026 - can be merged with alloc/page_index.h void free_registered(page_index *index, void *ptr); void release_page_index(page_index *index); void setup_page(uintptr_t address, uint8_t attributes); -void* palloc_inner(uint64_t size, uint8_t level, uint8_t attributes, bool full, bool map); +paddr_t palloc_inner(uint64_t size, uint8_t level, uint8_t attributes, bool full, bool map); void* palloc(uint64_t size, uint8_t level, uint8_t attributes, bool full); void free_managed_page(void* ptr); void pfree(void* ptr, uint64_t size); diff --git a/kernel/memory/talloc.c b/kernel/memory/talloc.c index 286fbffc..7076791e 100644 --- a/kernel/memory/talloc.c +++ b/kernel/memory/talloc.c @@ -2,6 +2,8 @@ #include "memory/mmu.h" #include "memory/page_allocator.h" #include "sysregs.h" +#include "memory/addr.h" +#include "memory/va_layout.h" #include "types.h" #include "exceptions/exception_handler.h" #include "console/kio.h" @@ -56,22 +58,58 @@ void* pre_talloc_ptr = 0; uintptr_t pre_talloc_mem_limit = 0; bool can_automap = false; +static bool talloc_high_va = false; void pre_talloc(){ - pre_talloc_ptr = palloc_inner(GRANULE_2MB, MEM_PRIV_KERNEL, MEM_DEV | MEM_RW, true, can_automap); + paddr_t phys = palloc_inner(GRANULE_2MB, MEM_PRIV_KERNEL, MEM_RW, true, can_automap); + if (!phys) panic("pre_talloc palloc failed", 0); + pre_talloc_ptr = (void*)phys; + uint64_t sctlr = 0; + asm volatile("mrs %0, sctlr_el1" : "=r"(sctlr)); + if ((sctlr & 1) != 0) pre_talloc_ptr = (void*)dmap_pa_to_kva(phys); pre_talloc_mem_limit = (uintptr_t)pre_talloc_ptr + GRANULE_2MB; - if (!can_automap){ + if (!can_automap || next_free_temp_memory == 0 || talloc_mem_limit == 0){ can_automap = true; next_free_temp_memory = (uintptr_t)pre_talloc_ptr; talloc_mem_limit = pre_talloc_mem_limit; } - mmu_map_all((uintptr_t)pre_talloc_ptr); + mmu_map_all(phys); +} + +void talloc_enable_high_va(){ + if (talloc_high_va) return; + + uint64_t sctlr = 0; + asm volatile("mrs %0, sctlr_el1" : "=r"(sctlr)); + if ((sctlr & 1) == 0) return; + + if (pre_talloc_ptr && (((uintptr_t)pre_talloc_ptr & HIGH_VA) != HIGH_VA)) pre_talloc_ptr = PHYS_TO_VIRT_P(pre_talloc_ptr); + if (pre_talloc_mem_limit && ((pre_talloc_mem_limit & HIGH_VA) != HIGH_VA)) pre_talloc_mem_limit = PHYS_TO_VIRT(pre_talloc_mem_limit); + if (next_free_temp_memory && ((next_free_temp_memory & HIGH_VA) != HIGH_VA)) next_free_temp_memory = PHYS_TO_VIRT(next_free_temp_memory); + if (talloc_mem_limit && ((talloc_mem_limit & HIGH_VA) != HIGH_VA)) talloc_mem_limit = PHYS_TO_VIRT(talloc_mem_limit); + + if (temp_free_list && (((uintptr_t)temp_free_list & HIGH_VA) != HIGH_VA)){ + temp_free_list = (FreeBlock*)PHYS_TO_VIRT((uintptr_t)temp_free_list); + FreeBlock* b = temp_free_list; + while (b && b->next && (((uintptr_t)b->next & HIGH_VA) != HIGH_VA)){ + b->next = (FreeBlock*)PHYS_TO_VIRT((uintptr_t)b->next); + b = b->next; + } + } + + talloc_high_va = true; } uint64_t talloc(uint64_t size){ size = (size + 0xFFF) & ~0xFFF; + if (!talloc_high_va) talloc_enable_high_va(); + + if (next_free_temp_memory == 0 || talloc_mem_limit == 0) { + if (!pre_talloc_ptr) pre_talloc(); + if (next_free_temp_memory == 0 || talloc_mem_limit == 0) panic("talloc not initialized", next_free_temp_memory); + } if (talloc_verbose){ uart_raw_puts("[talloc] Requested size: "); @@ -131,12 +169,12 @@ void temp_free(void* ptr, uint64_t size){ uart_raw_putc('\n'); } - memset(PHYS_TO_VIRT_P(ptr), 0, size); + memset(ptr, 0, size); - FreeBlock* block = VIRT_TO_PHYS_P(ptr); + FreeBlock* block = (FreeBlock*)ptr; block->size = size; block->next = temp_free_list; - temp_free_list = ptr; + temp_free_list = block; } void enable_talloc_verbose(){ @@ -151,7 +189,7 @@ uint64_t mem_get_kmem_end(){ return (uint64_t)&kcode_end; } -int handle_mem_node(const char *propname, const void *prop, uint32_t len, dtb_match_t *match){ +/* int handle_mem_node(const char *propname, const void *prop, uint32_t len, dtb_match_t *match){ if (strcmp(propname, "reg") == 0 && len >= 16){ uint32_t *p = (uint32_t *)prop; match->reg_base = ((uint64_t)__builtin_bswap32(p[0]) << 32) | __builtin_bswap32(p[1]); @@ -182,7 +220,7 @@ void calc_ram(){ if (USE_DTB && get_memory_region(&total_ram_start, &total_ram_size)){ calculated_ram_end = total_ram_start + total_ram_size; - calculated_ram_start = ((uint64_t)&kcode_end) + 0x1; + calculated_ram_start = ((uint64_t)&kcode_end) - KERNEL_IMAGE_VA_BASE + 0x1; calculated_ram_start = (calculated_ram_start + ((1ULL << 21) - 1)) & ~((1ULL << 21) - 1); calculated_ram_end = calculated_ram_end & ~((1ULL << 21) - 1); @@ -195,6 +233,16 @@ void calc_ram(){ calculated_ram_size = calculated_ram_end - calculated_ram_start; } +*/ +//TODO see dtb.c + +void calc_ram(){ + total_ram_start = RAM_START; + total_ram_size = RAM_SIZE; + calculated_ram_end = CRAM_END; + calculated_ram_start = CRAM_START; + calculated_ram_size = calculated_ram_end - calculated_ram_start; +} #define calcvar(var) \ if (var == 0) \ diff --git a/kernel/memory/talloc.h b/kernel/memory/talloc.h index 32347390..737da5bc 100644 --- a/kernel/memory/talloc.h +++ b/kernel/memory/talloc.h @@ -9,6 +9,7 @@ extern "C" { uint64_t talloc(uint64_t size); void temp_free(void* ptr, uint64_t size); void enable_talloc_verbose(); +void talloc_enable_high_va(); #ifdef __cplusplus } #endif diff --git a/kernel/memory/va_layout.h b/kernel/memory/va_layout.h new file mode 100644 index 00000000..ff57483a --- /dev/null +++ b/kernel/memory/va_layout.h @@ -0,0 +1,19 @@ +#pragma once + +#include "types.h" + +#include "memory/addr.h" + +#define KERNEL_IMAGE_VA_BASE 0xFFFFC00000000000ULL + +static inline paddr_t kimg_va_to_pa(kaddr_t va) { + return (paddr_t)((uint64_t)va - KERNEL_IMAGE_VA_BASE); +} + +static inline kaddr_t kimg_pa_to_va(paddr_t pa) { + return (kaddr_t)((uint64_t)pa + KERNEL_IMAGE_VA_BASE); +} + +static inline kaddr_t mmio_pa_to_kva(paddr_t pa) { + return (kaddr_t)((uint64_t)pa | HIGH_VA); +} diff --git a/kernel/networking/application_layer/http.c b/kernel/networking/application_layer/http.c index 42daf3af..eebf2698 100644 --- a/kernel/networking/application_layer/http.c +++ b/kernel/networking/application_layer/http.c @@ -14,7 +14,7 @@ string http_header_builder(const HTTPHeadersCommon *C, const HTTPHeader *H, uint string tmp = string_format("Content-Length: %i\r\n", (int)C->length); string_append_bytes(&out, tmp.data, tmp.length); - free_sized(tmp.data, tmp.mem_length); + string_free(tmp); if (C->date.length){ string_append_bytes(&out, "Date: ", 6); @@ -65,20 +65,20 @@ string http_header_builder(const HTTPHeadersCommon *C, const HTTPHeader *H, uint void http_headers_common_free(HTTPHeadersCommon *C){ if (!C) return; - if (C->type.mem_length) free_sized(C->type.data, C->type.mem_length); - if (C->date.mem_length) free_sized(C->date.data, C->date.mem_length); - if (C->connection.mem_length) free_sized(C->connection.data, C->connection.mem_length); - if (C->keep_alive.mem_length) free_sized(C->keep_alive.data, C->keep_alive.mem_length); - if (C->host.mem_length) free_sized(C->host.data, C->host.mem_length); - if (C->content_type.mem_length) free_sized(C->content_type.data, C->content_type.mem_length); + if (C->type.mem_length) string_free(C->type); + if (C->date.mem_length) string_free(C->date); + if (C->connection.mem_length) string_free(C->connection); + if (C->keep_alive.mem_length) string_free(C->keep_alive); + if (C->host.mem_length) string_free(C->host); + if (C->content_type.mem_length) string_free(C->content_type); *C = (HTTPHeadersCommon){0}; } void http_headers_extra_free(HTTPHeader *extra, uint32_t extra_count){ if (!extra) return; for (uint32_t i = 0; i < extra_count; i++){ - if (extra[i].key.mem_length) free_sized(extra[i].key.data, extra[i].key.mem_length); - if (extra[i].value.mem_length) free_sized(extra[i].value.data, extra[i].value.mem_length); + if (extra[i].key.mem_length) string_free(extra[i].key); + if (extra[i].value.mem_length) string_free(extra[i].value); } free_sized(extra, extra_count * sizeof(HTTPHeader)); } @@ -164,8 +164,8 @@ void http_header_parser(const char *buf, uint32_t len, if (extras && extra_i < max_lines){ extras[extra_i++] = (HTTPHeader){ key, value }; } else { - if (key.mem_length) free_sized(key.data, key.mem_length); - if (value.mem_length) free_sized(value.data, value.mem_length); + if (key.mem_length) string_free(key); + if (value.mem_length) string_free(value); } } @@ -195,8 +195,8 @@ void http_header_parser(const char *buf, uint32_t len, } for (uint32_t i = 0; i < extra_i; i++){ - if (extras[i].key.mem_length) free_sized(extras[i].key.data, extras[i].key.mem_length); - if (extras[i].value.mem_length) free_sized(extras[i].value.data, extras[i].value.mem_length); + if (extras[i].key.mem_length) string_free(extras[i].key); + if (extras[i].value.mem_length) string_free(extras[i].value); } free_sized(extras, sizeof(*extras) * max_lines); *out_extra = NULL; @@ -212,12 +212,12 @@ string http_request_builder(const HTTPRequestMsg *R){ string hdrs = http_header_builder(&R->headers_common, R->extra_headers, R->extra_header_count); string_append_bytes(&out, hdrs.data, hdrs.length); - free_sized(hdrs.data, hdrs.mem_length); + string_free(hdrs); if (R->body.ptr && R->body.size){ string body = string_from_literal_length((char*)R->body.ptr, R->body.size); string_append_bytes(&out, body.data, body.length); - free_sized(body.data, body.mem_length); + string_free(body); } return out; @@ -230,7 +230,7 @@ string http_response_builder(const HTTPResponseMsg *R){ string hdrs = http_header_builder(&R->headers_common, R->extra_headers, R->extra_header_count); string_append_bytes(&out, hdrs.data, hdrs.length); - free_sized(hdrs.data, hdrs.mem_length); + string_free(hdrs); if (R->body.ptr && R->body.size){ string_append_bytes(&out, (char*)R->body.ptr, (uint32_t)R->body.size); diff --git a/kernel/networking/application_layer/socket_http_client.hpp b/kernel/networking/application_layer/socket_http_client.hpp index d04b9610..be410252 100644 --- a/kernel/networking/application_layer/socket_http_client.hpp +++ b/kernel/networking/application_layer/socket_http_client.hpp @@ -101,7 +101,7 @@ class HTTPClient { } netlog_socket_event(&log_opts, &ev); - free_sized(out.data, out.mem_length); + string_free(out); if (sent < 0) { resp.status_code = (HttpError)sent; @@ -119,12 +119,12 @@ class HTTPClient { continue; } if (r < 0) { - free_sized(buf.data, buf.mem_length); + string_free(buf); resp.status_code = (HttpError)r; return resp; } if (r == 0) { - free_sized(buf.data, buf.mem_length); + string_free(buf); resp.status_code = (HttpError)SOCK_ERR_PROTO; return resp; } @@ -192,7 +192,7 @@ class HTTPClient { ev1.remote_ep = sock->get_remote_ep(); netlog_socket_event(&log_opts, &ev1); - free_sized(buf.data, buf.mem_length); + string_free(buf); return resp; } diff --git a/kernel/networking/application_layer/socket_http_server.hpp b/kernel/networking/application_layer/socket_http_server.hpp index fb8ed836..9e888288 100644 --- a/kernel/networking/application_layer/socket_http_server.hpp +++ b/kernel/networking/application_layer/socket_http_server.hpp @@ -91,7 +91,7 @@ class HTTPServer { continue; } if (r <= 0) { - free_sized(buf.data, buf.mem_length); + string_free(buf); return req; } string_append_bytes(&buf, tmp, (uint32_t)r); @@ -133,7 +133,7 @@ class HTTPServer { if (k < req.path.length) { string newp = string_repeat('\0', 0); string_append_bytes(&newp, req.path.data + k, req.path.length - k); - free_sized(req.path.data, req.path.mem_length); + string_free(req.path); req.path = newp; } } else if (req.path.length >= 8 && memcmp(req.path.data, "https://", 8) == 0) { @@ -142,7 +142,7 @@ class HTTPServer { if (k < req.path.length) { string newp = string_repeat('\0', 0); string_append_bytes(&newp, req.path.data + k, req.path.length - k); - free_sized(req.path.data, req.path.mem_length); + string_free(req.path); req.path = newp; } } @@ -204,7 +204,7 @@ class HTTPServer { netlog_socket_event(&log_opts, &ev); - free_sized(buf.data, buf.mem_length); + string_free(buf); return req; } @@ -240,7 +240,7 @@ class HTTPServer { ev.remote_ep = client->get_remote_ep(); netlog_socket_event(&log_opts, &ev); - free_sized(out.data, out.mem_length); + string_free(out); return sent < 0 ? (int32_t)sent : SOCK_OK; } diff --git a/kernel/networking/application_layer/ssdp.c b/kernel/networking/application_layer/ssdp.c index 99e2468e..96fa265b 100644 --- a/kernel/networking/application_layer/ssdp.c +++ b/kernel/networking/application_layer/ssdp.c @@ -63,7 +63,7 @@ string ssdp_build_notify(bool alive, bool v6) { HTTPHeadersCommon c = (HTTPHeadersCommon){0}; string hdrs = http_header_builder(&c, extra, 6); string_append_bytes(&out, hdrs.data, hdrs.length); - free_sized(hdrs.data, hdrs.mem_length); + string_free(hdrs); string_append_bytes(&out, "\r\n", 2); return out; } diff --git a/kernel/networking/application_layer/ssdp_daemon.c b/kernel/networking/application_layer/ssdp_daemon.c index eb3c0812..8771dcd5 100644 --- a/kernel/networking/application_layer/ssdp_daemon.c +++ b/kernel/networking/application_layer/ssdp_daemon.c @@ -54,7 +54,7 @@ static void ssdp_send_notify(socket_handle_t s4, socket_handle_t s6, bool alive) net_l4_endpoint dst; make_ep(ssdp_host_v4, 1900, IP_VER4, &dst); (void)socket_sendto_udp_ex(s4, DST_ENDPOINT, &dst, 0, msg.data, msg.length); - free_sized(msg.data, msg.mem_length); + string_free(msg); } if (s6) { @@ -64,7 +64,7 @@ static void ssdp_send_notify(socket_handle_t s4, socket_handle_t s6, bool alive) memcpy(dst.ip, ssdp_host_v6, 16); dst.port = 1900; (void)socket_sendto_udp_ex(s6, DST_ENDPOINT, &dst, 0, msg.data, msg.length); - free_sized(msg.data, msg.mem_length); + string_free(msg); } } @@ -160,7 +160,7 @@ int ssdp_daemon_entry(int argc, char* argv[]) { string resp = ssdp_build_search_response(); socket_handle_t sock = (ssdp_pending[i].dst.ver == IP_VER6) ? s6 : s4; if (sock) (void)socket_sendto_udp_ex(sock, DST_ENDPOINT, &ssdp_pending[i].dst, 0, resp.data, resp.length); - free_sized(resp.data, resp.mem_length); + string_free(resp); break; } diff --git a/kernel/networking/drivers/virtio_net_pci/virtio_net_pci.cpp b/kernel/networking/drivers/virtio_net_pci/virtio_net_pci.cpp index 601f9ee0..24b2cee9 100644 --- a/kernel/networking/drivers/virtio_net_pci/virtio_net_pci.cpp +++ b/kernel/networking/drivers/virtio_net_pci/virtio_net_pci.cpp @@ -12,11 +12,10 @@ #define TRANSMIT_QUEUE 1 #define CONTROL_QUEUE 2 -static constexpr uint32_t RX_BUF_SIZE = PAGE_SIZE; -static constexpr uint16_t RX_CHAIN_SEGS = 4; +constexpr uint32_t RX_BUF_SIZE = PAGE_SIZE; +constexpr uint16_t RX_CHAIN_SEGS = 4; -static void* g_rx_pool = nullptr; -static uint16_t g_rx_qsz = 0; +void* g_rx_pool = nullptr; #define kprintfv(fmt, ...) \ ({ \ @@ -44,7 +43,7 @@ typedef struct __attribute__((packed)) virtio_net_ctrl_ack_t { #define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 -static bool virtio_net_ctrl_send(virtio_device* dev, uint8_t cls, uint8_t cmd, const void* payload, uint32_t payload_len) { +bool virtio_net_ctrl_send(virtio_device* dev, uint8_t cls, uint8_t cmd, const void* payload, uint32_t payload_len) { if (!dev) return false; virtio_net_ctrl_hdr_t hdr; @@ -78,6 +77,12 @@ static bool virtio_net_ctrl_send(virtio_device* dev, uint8_t cls, uint8_t cmd, c } VirtioNetDriver::VirtioNetDriver() { + hw_name[0] = 0; +} + +VirtioNetDriver::~VirtioNetDriver(){} + +bool VirtioNetDriver::init_at(uint64_t addr, uint32_t irq_base_vector) { verbose = false; mrg_rxbuf = false; ctrl_vq = false; @@ -88,12 +93,12 @@ VirtioNetDriver::VirtioNetDriver() { duplex = LINK_DUPLEX_UNKNOWN; last_used_receive_idx = 0; last_used_sent_idx = 0; - hw_name[0] = 0; -} - -VirtioNetDriver::~VirtioNetDriver(){} + rx_desc = nullptr; + rx_avail = nullptr; + rx_used = nullptr; + rx_qsz = 0; + memset(&vnp_net_dev, 0, sizeof(vnp_net_dev)); -bool VirtioNetDriver::init_at(uint64_t addr, uint32_t irq_base_vector){ kprintfv("[virtio-net] probing pci_addr=%x",(uintptr_t)addr); uint64_t mmio_addr = 0, mmio_size = 0; @@ -139,23 +144,20 @@ bool VirtioNetDriver::init_at(uint64_t addr, uint32_t irq_base_vector){ ctrl_vq = (vnp_net_dev.negotiated_features & (1ULL << VIRTIO_NET_F_CTRL_VQ)) != 0; ctrl_rx = (vnp_net_dev.negotiated_features & (1ULL << VIRTIO_NET_F_CTRL_RX)) != 0; - if (ctrl_vq) { - select_queue(&vnp_net_dev, CONTROL_QUEUE); - if (!vnp_net_dev.common_cfg->queue_size) { - ctrl_vq = false; - ctrl_rx = false; - } else { - vnp_net_dev.common_cfg->queue_msix_vector = 0xFFFF; - } + if (CONTROL_QUEUE >= vnp_net_dev.num_queues || !vnp_net_dev.queues[CONTROL_QUEUE].valid || !vnp_net_dev.queues[CONTROL_QUEUE].size) { + ctrl_vq = false; + ctrl_rx = false; } - if (ctrl_vq && ctrl_rx) (void)sync_multicast((const uint8_t*)0, 0); - kprintfv("[virtio-net] negotiated ctrl_vq=%u ctrl_rx=%u", (unsigned)ctrl_vq, (unsigned)ctrl_rx); + if (RECEIVE_QUEUE >= vnp_net_dev.num_queues) return false; + if (!vnp_net_dev.queues[RECEIVE_QUEUE].valid) return false; - select_queue(&vnp_net_dev, RECEIVE_QUEUE); - uint16_t rx_qsz = vnp_net_dev.common_cfg->queue_size; - if (!rx_qsz) return false; - g_rx_qsz = rx_qsz; + rx_qsz = vnp_net_dev.queues[RECEIVE_QUEUE].size; + rx_desc = vnp_net_dev.queues[RECEIVE_QUEUE].desc; + rx_avail = vnp_net_dev.queues[RECEIVE_QUEUE].driver; + rx_used = vnp_net_dev.queues[RECEIVE_QUEUE].device; + + if (!rx_qsz || !rx_desc || !rx_avail || !rx_used) return false; kprintfv("[virtio-net] RX qsz=%u",rx_qsz); if (!g_rx_pool){ @@ -163,13 +165,15 @@ bool VirtioNetDriver::init_at(uint64_t addr, uint32_t irq_base_vector){ kprintfv("[virtio-net] rx_pool=%x",(uintptr_t)g_rx_pool); if (!g_rx_pool) return false; } - volatile virtq_desc* rx_desc = (volatile virtq_desc*)PHYS_TO_VIRT_P((virtq_desc*)vnp_net_dev.common_cfg->queue_desc); - volatile virtq_avail* rx_avail = (volatile virtq_avail*)PHYS_TO_VIRT_P((virtq_avail*)vnp_net_dev.common_cfg->queue_driver); uint16_t chain_count = (uint16_t)(rx_qsz / RX_CHAIN_SEGS); if (!chain_count) return false; + memset((void*)rx_desc, 0, 16ULL * rx_qsz); + rx_avail->flags = 0; rx_avail->idx = 0; + rx_used->flags = 0; + rx_used->idx = 0; for (uint16_t c = 0; c < chain_count; c++) { uint16_t head = (uint16_t)(c * RX_CHAIN_SEGS); @@ -191,20 +195,33 @@ bool VirtioNetDriver::init_at(uint64_t addr, uint32_t irq_base_vector){ } asm volatile ("dmb ishst" ::: "memory"); - virtio_notify(&vnp_net_dev); + select_queue(&vnp_net_dev, RECEIVE_QUEUE); vnp_net_dev.common_cfg->queue_msix_vector = 0; kprintfv("[virtio-net] RX vector=%u",vnp_net_dev.common_cfg->queue_msix_vector); if (vnp_net_dev.common_cfg->queue_msix_vector != 0) return false; + virtio_notify(&vnp_net_dev); + + if (TRANSMIT_QUEUE >= vnp_net_dev.num_queues) return false; + if (!vnp_net_dev.queues[TRANSMIT_QUEUE].valid) return false; select_queue(&vnp_net_dev, TRANSMIT_QUEUE); vnp_net_dev.common_cfg->queue_msix_vector = 1; kprintfv("[virtio-net] TX vector=%u",vnp_net_dev.common_cfg->queue_msix_vector); if (vnp_net_dev.common_cfg->queue_msix_vector != 1) return false; - virtio_net_config* cfg = (virtio_net_config*)vnp_net_dev.device_cfg; + if (ctrl_vq) { + select_queue(&vnp_net_dev, CONTROL_QUEUE); + vnp_net_dev.common_cfg->queue_msix_vector = 0xFFFF; + } + + if (ctrl_vq && ctrl_rx) (void)sync_multicast((const uint8_t*)0, 0); + kprintfv("[virtio-net] negotiated ctrl_vq=%u ctrl_rx=%u", (unsigned)ctrl_vq, (unsigned)ctrl_rx); - uint8_t mac[6]; get_mac(mac); + volatile virtio_net_config* cfg = (volatile virtio_net_config*)vnp_net_dev.device_cfg; + + uint8_t mac[6]; + get_mac(mac); uint16_t dev_mtu = cfg->mtu; @@ -231,14 +248,16 @@ bool VirtioNetDriver::init_at(uint64_t addr, uint32_t irq_base_vector){ (unsigned)mtu, (unsigned)header_size, dpx_str); } + vnp_net_dev.common_cfg->device_status |= VIRTIO_STATUS_DRIVER_OK; + select_queue(&vnp_net_dev, RECEIVE_QUEUE); return true; } void VirtioNetDriver::get_mac(uint8_t out_mac[6]) const { - virtio_net_config* net_config = (virtio_net_config*)vnp_net_dev.device_cfg; - memcpy(out_mac, net_config->mac, 6); + volatile virtio_net_config* net_config = (volatile virtio_net_config*)vnp_net_dev.device_cfg; + memcpy(out_mac, (const void*)net_config->mac, 6); } uint16_t VirtioNetDriver::get_mtu() const { @@ -275,12 +294,11 @@ sizedptr VirtioNetDriver::handle_receive_packet(){ disable_interrupt(); select_queue(&vnp_net_dev, RECEIVE_QUEUE); - volatile virtq_used* used = (volatile virtq_used*)PHYS_TO_VIRT_P((void*)(uintptr_t)vnp_net_dev.common_cfg->queue_device); - volatile virtq_desc* desc = (volatile virtq_desc*)PHYS_TO_VIRT_P((void*)(uintptr_t)vnp_net_dev.common_cfg->queue_desc); - volatile virtq_avail* avail = (volatile virtq_avail*)PHYS_TO_VIRT_P((void*)(uintptr_t)vnp_net_dev.common_cfg->queue_driver); - - uint16_t qsz = vnp_net_dev.common_cfg->queue_size; - if (!qsz) { + volatile virtq_used* used = rx_used; + volatile virtq_desc* desc = rx_desc; + volatile virtq_avail* avail = rx_avail; + uint16_t qsz = rx_qsz; + if (!qsz || !used || !desc || !avail) { enable_interrupt(); return (sizedptr){0,0}; } @@ -304,6 +322,7 @@ sizedptr VirtioNetDriver::handle_receive_packet(){ asm volatile ("dmb ishst" ::: "memory"); avail->idx = (uint16_t)(aidx + 1); asm volatile ("dmb ishst" ::: "memory"); + select_queue(&vnp_net_dev, RECEIVE_QUEUE); virtio_notify(&vnp_net_dev); enable_interrupt(); return (sizedptr){0,0}; @@ -320,6 +339,7 @@ sizedptr VirtioNetDriver::handle_receive_packet(){ asm volatile ("dmb ishst" ::: "memory"); avail->idx = (uint16_t)(aidx + 1); asm volatile ("dmb ishst" ::: "memory"); + select_queue(&vnp_net_dev, RECEIVE_QUEUE); virtio_notify(&vnp_net_dev); enable_interrupt(); return (sizedptr){0,0}; @@ -337,6 +357,7 @@ sizedptr VirtioNetDriver::handle_receive_packet(){ asm volatile ("dmb ishst" ::: "memory"); avail->idx = (uint16_t)(aidx + 1); asm volatile ("dmb ishst" ::: "memory"); + select_queue(&vnp_net_dev, RECEIVE_QUEUE); virtio_notify(&vnp_net_dev); enable_interrupt(); return (sizedptr){0,0}; @@ -369,6 +390,7 @@ sizedptr VirtioNetDriver::handle_receive_packet(){ asm volatile ("dmb ishst" ::: "memory"); avail->idx = (uint16_t)(aidx + 1); asm volatile ("dmb ishst" ::: "memory"); + select_queue(&vnp_net_dev, RECEIVE_QUEUE); virtio_notify(&vnp_net_dev); enable_interrupt(); @@ -381,10 +403,9 @@ sizedptr VirtioNetDriver::handle_receive_packet(){ } void VirtioNetDriver::handle_sent_packet(){ - select_queue(&vnp_net_dev, TRANSMIT_QUEUE); - - volatile virtq_used* used = (volatile virtq_used*)PHYS_TO_VIRT_P((void*)(uintptr_t)vnp_net_dev.common_cfg->queue_device); - last_used_sent_idx = used->idx; + if (TRANSMIT_QUEUE >= vnp_net_dev.num_queues) return; + if (!vnp_net_dev.queues[TRANSMIT_QUEUE].device) return; + last_used_sent_idx = vnp_net_dev.queues[TRANSMIT_QUEUE].device->idx; } bool VirtioNetDriver::send_packet(sizedptr packet){ @@ -403,7 +424,7 @@ bool VirtioNetDriver::send_packet(sizedptr packet){ enable_interrupt(); kprintfv("[virtio-net] tx queued len=%u",(unsigned)packet.size); - kfree((void*)packet.ptr, packet.size); + if (ok) kfree((void*)packet.ptr, packet.size); return ok; } diff --git a/kernel/networking/drivers/virtio_net_pci/virtio_net_pci.hpp b/kernel/networking/drivers/virtio_net_pci/virtio_net_pci.hpp index 2b6e3501..19b8b4fb 100644 --- a/kernel/networking/drivers/virtio_net_pci/virtio_net_pci.hpp +++ b/kernel/networking/drivers/virtio_net_pci/virtio_net_pci.hpp @@ -73,25 +73,25 @@ class VirtioNetDriver : public NetDriver { bool send_packet(sizedptr packet) override; private: - virtio_device vnp_net_dev; + virtio_device vnp_net_dev = {}; - volatile virtq_desc* rx_desc; - volatile virtq_avail* rx_avail; - volatile virtq_used* rx_used; - uint16_t rx_qsz; + volatile virtq_desc* rx_desc = nullptr; + volatile virtq_avail* rx_avail = nullptr; + volatile virtq_used* rx_used = nullptr; + uint16_t rx_qsz = 0; - bool verbose; - bool mrg_rxbuf; + bool verbose = false; + bool mrg_rxbuf = false; - bool ctrl_vq; - bool ctrl_rx; + bool ctrl_vq = false; + bool ctrl_rx = false; - uint16_t header_size; - uint16_t mtu; - uint32_t speed_mbps; - LinkDuplex duplex; - char hw_name[8]; + uint16_t header_size = sizeof(virtio_net_hdr_t); + uint16_t mtu = 1500; + uint32_t speed_mbps = 0xFFFFFFFFu; + LinkDuplex duplex = LINK_DUPLEX_UNKNOWN; + char hw_name[8] = {}; - uint16_t last_used_receive_idx; - uint16_t last_used_sent_idx; + uint16_t last_used_receive_idx = 0; + uint16_t last_used_sent_idx = 0; }; diff --git a/kernel/networking/internet_layer/ipv4.c b/kernel/networking/internet_layer/ipv4.c index 88aeb058..a8e7edf5 100644 --- a/kernel/networking/internet_layer/ipv4.c +++ b/kernel/networking/internet_layer/ipv4.c @@ -17,7 +17,7 @@ static uint16_t g_ip_ident = 1; static l3_ipv4_interface_t* best_v4_on_l2_for_dst(l2_interface_t* l2, uint32_t dst) { l3_ipv4_interface_t* best = NULL; - uint32_t best_mask = 0; + uint32_t best_pl = -1; for (int s = 0; s < MAX_IPV4_PER_INTERFACE; s++) { l3_ipv4_interface_t* v4 = l2->l3_v4[s]; if (!v4) continue; @@ -25,13 +25,13 @@ static l3_ipv4_interface_t* best_v4_on_l2_for_dst(l2_interface_t* l2, uint32_t d if (!v4->ip) continue; uint32_t m = v4->mask; if (m && (ipv4_net(dst, m) == ipv4_net(v4->ip, m))) { - if (!best || m > best_mask) { + uint32_t pl = ipv4_prefix_len(m); + if (pl > best_pl) { + best_pl = pl; best = v4; - best_mask = m; } } else if (!best) { best = v4; - best_mask = m; } } return best; @@ -82,7 +82,8 @@ static bool pick_broadcast_bound_l3(uint8_t l3_id, uint8_t* out_ifx, uint32_t* o if (v4->mode == IPV4_CFG_DISABLED) return false; if (out_ifx) *out_ifx = v4->l2->ifindex; - if (out_src) *out_src = v4->ip ? v4->ip : 0; + if (out_src) *out_src = v4->ip; + if (!v4->ip && v4->mode == IPV4_CFG_DHCP && out_src) *out_src = 0; if (out_nh) *out_nh = 0xFFFFFFFFu; return true; } @@ -138,7 +139,7 @@ static bool pick_broadcast_global(uint8_t* out_ifx, uint32_t* out_src, uint32_t* } static bool pick_route_global(uint32_t dst, uint8_t* out_ifx, uint32_t* out_src, uint32_t* out_nh) { - if (dst == 0xFFFFFFFFu) return pick_broadcast_global(out_ifx, out_src, out_nh); + if (ipv4_is_limited_broadcast(dst)) return pick_broadcast_global(out_ifx, out_src, out_nh); ip_resolution_result_t r = resolve_ipv4_to_interface(dst); if (r.found && r.ipv4 && r.l2) { @@ -169,7 +170,7 @@ static bool pick_route_global(uint32_t dst, uint8_t* out_ifx, uint32_t* out_src, } static bool pick_route_bound_l3(uint8_t l3_id, uint32_t dst, uint8_t* out_ifx, uint32_t* out_src, uint32_t* out_nh) { - if (dst == 0xFFFFFFFFu) return pick_broadcast_bound_l3(l3_id, out_ifx, out_src, out_nh); + if (ipv4_is_limited_broadcast(dst)) return pick_broadcast_bound_l3(l3_id, out_ifx, out_src, out_nh); l3_ipv4_interface_t* v4 = l3_ipv4_find_by_id(l3_id); if (!v4 || !v4->l2) return false; @@ -202,7 +203,7 @@ static bool pick_route_bound_l3(uint8_t l3_id, uint32_t dst, uint8_t* out_ifx, u } static bool pick_route_bound_l2(uint8_t ifindex, uint32_t dst, uint8_t* out_ifx, uint32_t* out_src, uint32_t* out_nh) { - if (dst == 0xFFFFFFFFu) return pick_broadcast_bound_l2(ifindex, out_ifx, out_src, out_nh); + if (ipv4_is_limited_broadcast(dst)) return pick_broadcast_bound_l2(ifindex, out_ifx, out_src, out_nh); l2_interface_t* l2 = l2_interface_find_by_index(ifindex); if (!l2) return false; diff --git a/kernel/networking/netpkt.c b/kernel/networking/netpkt.c index 3f6f1167..dce4093b 100644 --- a/kernel/networking/netpkt.c +++ b/kernel/networking/netpkt.c @@ -1,82 +1,496 @@ #include "netpkt.h" #include "std/std.h" -#include "syscalls/syscalls.h" +#include "memory/page_allocator.h" -struct netpkt { +#define NETPKT_F_VIEW 1u +#define NETPKT_BUF_F_EXTERNAL 1u +#define NETPKT_META_KEEP_EMPTY_PAGES 2u + +typedef struct netpkt_buf netpkt_buf_t; + +typedef struct meta_slab_page meta_slab_page_t; + +typedef struct meta_obj_hdr { + meta_slab_page_t*page; + uintptr_t pad; +} meta_obj_hdr_t; + +struct meta_slab_page { + meta_slab_page_t* next; + void* free_list; + uint16_t in_use; + uint16_t capacity; + uint32_t stride; +}; + +typedef struct meta_slab { + meta_slab_page_t* partial; + meta_slab_page_t* full; + meta_slab_page_t* empty; + uint32_t obj_size; + uint32_t stride; + uint32_t empty_pages; + uint32_t keep_empty; +} meta_slab_t; + +struct netpkt_buf { uintptr_t base; uint32_t alloc; - uint32_t head; - uint32_t len; uint32_t refs; + uint32_t flags; netpkt_free_fn free_fn; void* free_ctx; }; -static void netpkt_free_malloc(void* ctx, uintptr_t base, uint32_t alloc_size) { - (void)ctx; - if (base && alloc_size) free_sized((void*)base, alloc_size); +struct netpkt { + netpkt_buf_t* buf; + uint32_t off; + uint32_t cap; + uint32_t head; + uint32_t len; + uint32_t refs; + uint32_t flags; +}; + +static uint64_t g_netpkt_page_bytes; +static uint64_t g_netpkt_payload_page_bytes; +static uint64_t g_netpkt_meta_page_bytes; + +static meta_slab_t g_meta_slab_pkt; +static meta_slab_t g_meta_slab_buf; + +static uintptr_t g_spare_page; + +static void* meta_slab_alloc(meta_slab_t* s, uint32_t obj_size_aligned) { + if (!s) return 0; + if (!obj_size_aligned) return 0; + if ((obj_size_aligned & 15u) != 0u) return 0; + + if (!s->obj_size) { + s->obj_size = obj_size_aligned; + s->stride = (uint32_t)(((uint64_t)sizeof(meta_obj_hdr_t) + (uint64_t)obj_size_aligned + 15ull) & ~15ull); + if (!s->stride) return 0; + s->keep_empty = NETPKT_META_KEEP_EMPTY_PAGES; + } else { + if (s->obj_size != obj_size_aligned) return 0; + if (!s->stride) return 0; + } + + if (!s->partial) { + if (s->empty) { + meta_slab_page_t* p =s->empty; + s->empty = p->next; + if (s->empty_pages) s->empty_pages--; + p->next = s->partial; + s->partial = p; + } else { + if ((uint64_t)PAGE_SIZE > (uint64_t)NETPKT_MAX_PAGE_BYTES) return 0; + if (g_netpkt_page_bytes > (uint64_t)NETPKT_MAX_PAGE_BYTES - (uint64_t)PAGE_SIZE) return 0; + + g_netpkt_page_bytes += (uint64_t)PAGE_SIZE; + g_netpkt_meta_page_bytes += (uint64_t)PAGE_SIZE; + + meta_slab_page_t* p = (meta_slab_page_t*)palloc(PAGE_SIZE, MEM_PRIV_KERNEL, MEM_RW | MEM_NORM, true); + if (!p) { + if (g_netpkt_page_bytes >= (uint64_t)PAGE_SIZE) g_netpkt_page_bytes-= (uint64_t)PAGE_SIZE; + else g_netpkt_page_bytes = 0; + + if (g_netpkt_meta_page_bytes >= (uint64_t)PAGE_SIZE) g_netpkt_meta_page_bytes -= (uint64_t)PAGE_SIZE; + else g_netpkt_meta_page_bytes = 0; + return 0; + } + + p->next = 0;// + p->free_list = 0; + p->in_use = 0; + p->capacity = 0; + p->stride = s->stride; + + uint8_t* start = (uint8_t*)p + (uint64_t)sizeof(*p); + start = (uint8_t*)(((uintptr_t)start + 15u) &~(uintptr_t)15u); + + uint64_t avail = (uint64_t)PAGE_SIZE - (uint64_t)(start - (uint8_t*)p); + if (avail < (uint64_t)p->stride) { + pfree(p, (uint64_t)PAGE_SIZE); + if (g_netpkt_page_bytes >= (uint64_t)PAGE_SIZE) g_netpkt_page_bytes -= (uint64_t)PAGE_SIZE; + else g_netpkt_page_bytes = 0; + + if (g_netpkt_meta_page_bytes >= (uint64_t)PAGE_SIZE) g_netpkt_meta_page_bytes -= (uint64_t)PAGE_SIZE; + else g_netpkt_meta_page_bytes = 0; + return 0; + } + + uint32_t cap = (uint32_t)(avail / (uint64_t)p->stride); + if (!cap) { + pfree(p, (uint64_t)PAGE_SIZE); + if (g_netpkt_page_bytes >= (uint64_t)PAGE_SIZE) g_netpkt_page_bytes -= (uint64_t)PAGE_SIZE; + else g_netpkt_page_bytes = 0; + if (g_netpkt_meta_page_bytes >= (uint64_t)PAGE_SIZE) g_netpkt_meta_page_bytes -= (uint64_t)PAGE_SIZE; + else g_netpkt_meta_page_bytes = 0; + return 0; + } + + p->capacity = (uint16_t)cap; + + uint32_t i = 0; + while (i< cap) { + uint8_t* slot = start + (uint64_t)i * (uint64_t)p->stride; + meta_obj_hdr_t* h = (meta_obj_hdr_t*)(uintptr_t)slot; + h->page = p; + h->pad = 0; + void* obj = (void*)(uintptr_t)(slot + (uint64_t)sizeof(*h)); + + *(void**)obj = p->free_list; + p->free_list = obj; + i++; + } + + p->next = s->partial; + s->partial = p; + } + } + + meta_slab_page_t* page = s->partial; + if (!page) return 0; + if (!page->free_list) return 0; + + void* obj = page->free_list; + page->free_list = *(void**)obj; + page->in_use++; + + if (page->in_use == page->capacity) { + s->partial = page->next; + page->next = s->full; + s->full = page; + } + + return obj; +} + +static void meta_slab_free(meta_slab_t* s, void* obj) { + if (!s) return; + if (!obj) return; + + meta_obj_hdr_t* h = (meta_obj_hdr_t*)(uintptr_t)obj -1; + meta_slab_page_t* page = h->page; + if (!page) return; + + *(void**)obj = page->free_list; + page->free_list = obj; + + bool was_full = false; + if (page->in_use == page->capacity) was_full = true; + if (page->in_use) page->in_use--; + + if (was_full) { + meta_slab_page_t* prev = 0; + meta_slab_page_t* cur = s->full; + while (cur && cur != page) { + prev = cur; + cur = cur->next; //minchia fondamenti di informatica + } + if (cur) { + if (prev) prev->next = cur->next; + else s->full = cur->next; + cur->next = s->partial; + s->partial = cur; + } + } + + if (!page->in_use) { + meta_slab_page_t* prev = 0; + meta_slab_page_t* cur = s->partial; + while (cur && cur != page) { + prev = cur; + cur = cur->next; + } + if (cur) { + if (prev) prev->next = cur->next; + else s->partial = cur->next; + } else { + prev = 0; + cur = s->full; + while (cur && cur != page) { + prev = cur; + cur = cur->next; + } + if (cur) { + if (prev) prev->next = cur->next; + else s->full = cur->next; + } + } + + page->next = s->empty; + s->empty = page; + s->empty_pages++; + + while (s->empty_pages > s->keep_empty){ + meta_slab_page_t* e = s->empty; + if (!e) break; + s->empty = e->next; + s->empty_pages--; + pfree(e,(uint64_t)PAGE_SIZE); + if (g_netpkt_page_bytes >= (uint64_t)PAGE_SIZE) g_netpkt_page_bytes -= (uint64_t)PAGE_SIZE; + else g_netpkt_page_bytes = 0; + if (g_netpkt_meta_page_bytes >= (uint64_t)PAGE_SIZE) g_netpkt_meta_page_bytes -= (uint64_t)PAGE_SIZE; + else g_netpkt_meta_page_bytes = 0; + } + } } static bool netpkt_realloc_to(netpkt_t* p, uint32_t new_head, uint32_t new_alloc) { if (!p) return false; - if (new_alloc < new_head + p->len) return false; + if (!p->buf) return false; + if (p->flags & NETPKT_F_VIEW) return false; + + uint64_t min = (uint64_t)new_head + (uint64_t)p->len; + if ((uint64_t)new_alloc < min) return false; + + uint64_t bytes = (uint64_t)new_alloc; + if (!bytes) bytes = 1; + uint64_t cap64 = count_pages(bytes, PAGE_SIZE)*(uint64_t)PAGE_SIZE; + if (!cap64) cap64 = PAGE_SIZE; + if (cap64 > (uint64_t)NETPKT_MAX_ALLOC) return false; + + uint32_t cap = (uint32_t)cap64; + if ((uint64_t)new_head+(uint64_t)p->len > (uint64_t)cap) return false; + + void* mem = 0; + bool from_spare = false; + if (cap == PAGE_SIZE && g_spare_page) { + mem = (void*)g_spare_page; + g_spare_page = 0; + from_spare = true; + memset(mem, 0, PAGE_SIZE); + } else { + if ((uint64_t)cap > (uint64_t)NETPKT_MAX_PAGE_BYTES) return false; + if (g_netpkt_page_bytes > (uint64_t)NETPKT_MAX_PAGE_BYTES - (uint64_t)cap)return false; + g_netpkt_page_bytes += (uint64_t)cap; + g_netpkt_payload_page_bytes += (uint64_t)cap; + + mem = palloc((uint64_t)cap, MEM_PRIV_KERNEL, MEM_RW | MEM_NORM, true); + if (!mem) { + if (g_netpkt_page_bytes >= (uint64_t)cap) g_netpkt_page_bytes -= (uint64_t)cap; + else g_netpkt_page_bytes = 0; + + if (g_netpkt_payload_page_bytes >= (uint64_t)cap) g_netpkt_payload_page_bytes -= (uint64_t)cap; + else g_netpkt_payload_page_bytes = 0; + return false; + } + } - uintptr_t nb = (uintptr_t)malloc(new_alloc); - if (!nb) return false; + uint32_t sz = (uint32_t)(((uint64_t)sizeof(netpkt_buf_t) + 15) & ~15ull); + netpkt_buf_t* nb = (netpkt_buf_t*)meta_slab_alloc(&g_meta_slab_buf, sz); + if (!nb) { + if (from_spare) { + g_spare_page = (uintptr_t)mem; + } else { + pfree(mem, (uint64_t)cap); + if (g_netpkt_page_bytes >= (uint64_t)cap) g_netpkt_page_bytes -= (uint64_t)cap; + else g_netpkt_page_bytes = 0; + if (g_netpkt_payload_page_bytes >= (uint64_t)cap) g_netpkt_payload_page_bytes -= (uint64_t)cap; + else g_netpkt_payload_page_bytes = 0; + } + return false; + } + + nb->base = (uintptr_t)mem; + nb->alloc = cap; + nb->refs = 1; + nb->flags = 0; + nb->free_fn = 0; + nb->free_ctx = 0; - if (p->len) memcpy((void*)(nb + new_head), (const void*)(p->base + p->head), p->len); - if (p->free_fn) p->free_fn(p->free_ctx, p->base, p->alloc); + if (p->len) memcpy((void*)(nb->base + (uintptr_t)new_head), (const void*)netpkt_data(p), p->len); - p->base = nb; - p->alloc = new_alloc; + netpkt_buf_t* ob = p->buf; + p->buf = nb; + p->off = 0; + p->cap = cap; p->head = new_head; - if (!p->free_fn) p->free_fn = netpkt_free_malloc; - p->free_ctx = 0; + + if (ob) { + if (ob->refs > 1) { + ob->refs--; + } else { + if (ob->flags & NETPKT_BUF_F_EXTERNAL) { + if (ob->free_fn) ob->free_fn(ob->free_ctx, ob->base, ob->alloc); + } else { + bool aligned = ((ob->base & (PAGE_SIZE - 1)) == 0) && ((ob->alloc & (PAGE_SIZE - 1)) == 0); + if (!g_spare_page && aligned) { + g_spare_page = ob->base; + + if (ob->alloc > PAGE_SIZE) { + pfree((void*)(ob->base + PAGE_SIZE), (uint64_t)ob->alloc - (uint64_t)PAGE_SIZE); + uint64_t dec = (uint64_t)ob->alloc-(uint64_t)PAGE_SIZE; + if (g_netpkt_page_bytes >= dec) g_netpkt_page_bytes -= dec; + else g_netpkt_page_bytes = 0; + if (g_netpkt_payload_page_bytes >= dec) g_netpkt_payload_page_bytes -= dec; + else g_netpkt_payload_page_bytes = 0; + } + } else { + if (ob->base) pfree((void*)ob->base,(uint64_t)ob->alloc); + uint64_t dec = (uint64_t)ob->alloc; + if (g_netpkt_page_bytes >= dec) g_netpkt_page_bytes -= dec; + else g_netpkt_page_bytes = 0; + if (g_netpkt_payload_page_bytes >= dec) g_netpkt_payload_page_bytes -= dec; + else g_netpkt_payload_page_bytes = 0; + } + } + + meta_slab_free(&g_meta_slab_buf, ob); + } + } + return true; } netpkt_t* netpkt_alloc(uint32_t data_capacity, uint32_t headroom, uint32_t tailroom) { - uint32_t alloc = headroom+ data_capacity + tailroom; - if (alloc == 0) alloc = 1; + uint64_t alloc = (uint64_t)headroom + (uint64_t)data_capacity + (uint64_t)tailroom; + if (alloc > (uint64_t)NETPKT_MAX_ALLOC) return 0; - uintptr_t base = (uintptr_t)malloc(alloc); - if (!base) return 0; + if (!alloc) alloc = 1; + uint64_t cap64 = count_pages(alloc, PAGE_SIZE) * (uint64_t)PAGE_SIZE; + if (!cap64) cap64 = PAGE_SIZE; + if (cap64 > (uint64_t)NETPKT_MAX_ALLOC) return 0; + + uint32_t cap = (uint32_t)cap64; + + void* mem = 0; + bool from_spare = false; + if (cap == PAGE_SIZE && g_spare_page) { + mem = (void*)g_spare_page; + g_spare_page = 0; + from_spare = true; + memset(mem, 0, PAGE_SIZE); + } else { + if ((uint64_t)cap > (uint64_t)NETPKT_MAX_PAGE_BYTES) return 0; + if (g_netpkt_page_bytes > (uint64_t)NETPKT_MAX_PAGE_BYTES - (uint64_t)cap) return 0; + g_netpkt_page_bytes += (uint64_t)cap; + g_netpkt_payload_page_bytes += (uint64_t)cap; + + mem = palloc((uint64_t)cap, MEM_PRIV_KERNEL, MEM_RW | MEM_NORM, true); + if (!mem) { + if (g_netpkt_page_bytes >= (uint64_t)cap) g_netpkt_page_bytes -= (uint64_t)cap; + else g_netpkt_page_bytes = 0; + if (g_netpkt_payload_page_bytes >= (uint64_t)cap) g_netpkt_payload_page_bytes -= (uint64_t)cap; + else g_netpkt_payload_page_bytes = 0; + return 0; + } + } + + uint32_t bsz = (uint32_t)(((uint64_t)sizeof(netpkt_buf_t) + 15ull) &~15ull); + netpkt_buf_t* b = (netpkt_buf_t*)meta_slab_alloc(&g_meta_slab_buf, bsz); + if (!b) { + if (from_spare) { + g_spare_page = (uintptr_t)mem; + } else { + pfree(mem, (uint64_t)cap); + if (g_netpkt_page_bytes >= (uint64_t)cap) g_netpkt_page_bytes -= (uint64_t)cap; + else g_netpkt_page_bytes = 0; + if (g_netpkt_payload_page_bytes >= (uint64_t)cap) g_netpkt_payload_page_bytes -= (uint64_t)cap; + else g_netpkt_payload_page_bytes = 0; + } + return 0; + } - netpkt_t* p = (netpkt_t*)malloc(sizeof(netpkt_t)); + b->base = (uintptr_t)mem; + b->alloc = cap; + b->refs = 1; + b->flags = 0; + b->free_fn = 0; + b->free_ctx = 0; + + uint32_t psz = (uint32_t)(((uint64_t)sizeof(netpkt_t) + 15ull) &~15ull); + netpkt_t* p = (netpkt_t*)meta_slab_alloc(&g_meta_slab_pkt, psz); if (!p) { - free_sized((void*)base, alloc); + meta_slab_free(&g_meta_slab_buf, b); + if (from_spare) { + g_spare_page = (uintptr_t)mem; + } else { + pfree(mem, (uint64_t)cap); + if (g_netpkt_page_bytes >= (uint64_t)cap) g_netpkt_page_bytes -= (uint64_t)cap; + else g_netpkt_page_bytes = 0; + if (g_netpkt_payload_page_bytes >= (uint64_t)cap) g_netpkt_payload_page_bytes -= (uint64_t)cap; + else g_netpkt_payload_page_bytes = 0; + } return 0; } - p->base = base; - p->alloc = alloc; + p->buf = b; + p->off = 0; + p->cap = cap; p->head = headroom; p->len = 0; p->refs = 1; - p->free_fn = netpkt_free_malloc; - p->free_ctx = 0; + p->flags = 0; return p; } -netpkt_t* netpkt_wrap(uintptr_t base, uint32_t alloc_size, uint32_t data_len, netpkt_free_fn free_fn, void* ctx) { - if (!base || !alloc_size) return 0; - if (data_len > alloc_size) return 0; +netpkt_t* netpkt_wrap(uintptr_t base, uint32_t alloc_size, uint32_t data_off, uint32_t data_len, netpkt_free_fn free_fn, void* ctx) { + if (!base) return 0; + if (!alloc_size) return 0; + if (alloc_size > NETPKT_MAX_ALLOC) return 0; + if (data_off > alloc_size) return 0; + if (data_len > alloc_size - data_off) return 0; + + uint32_t bsz = (uint32_t)(((uint64_t)sizeof(netpkt_buf_t) + 15ull) &~15ull); + netpkt_buf_t* b = (netpkt_buf_t*)meta_slab_alloc(&g_meta_slab_buf, bsz); + if (!b) return 0; - netpkt_t* p = (netpkt_t*)malloc(sizeof(netpkt_t)); - if (!p) return 0; + b->base = base; + b->alloc = alloc_size; + b->refs = 1; + b->flags = NETPKT_BUF_F_EXTERNAL; + b->free_fn = free_fn; + b->free_ctx = ctx; + + uint32_t psz = (uint32_t)(((uint64_t)sizeof(netpkt_t) + 15ull) &~15ull); + netpkt_t* p = (netpkt_t*)meta_slab_alloc(&g_meta_slab_pkt, psz); + if (!p) { + meta_slab_free(&g_meta_slab_buf, b); + return 0; + } - p->base = base; - p->alloc = alloc_size; - p->head = 0; + p->buf = b; + p->off = 0; + p->cap = alloc_size; + p->head = data_off; p->len = data_len; p->refs = 1; - p->free_fn = free_fn ? free_fn : netpkt_free_malloc; - p->free_ctx = ctx; + p->flags = 0; return p; } +netpkt_t* netpkt_view(netpkt_t* parent, uint32_t off, uint32_t len) { + if (!parent) return 0; + if (!parent->buf) return 0; + + uint64_t end = (uint64_t)off+(uint64_t)len; + if (end > (uint64_t)parent->len) return 0; + + uint64_t abs = (uint64_t)parent->off + (uint64_t)parent->head + (uint64_t)off; + if (abs + (uint64_t)len > (uint64_t)parent->buf->alloc) return 0; + + uint32_t psz = (uint32_t)(((uint64_t)sizeof(netpkt_t) + 15ull) &~15ull); + netpkt_t* v = (netpkt_t*)meta_slab_alloc(&g_meta_slab_pkt, psz); + if (!v) return 0; + + parent->buf->refs++; + + v->buf = parent->buf; + v->off = (uint32_t)abs; + v->cap = len; + v->head = 0; + v->len = len; + v->refs = 1; + v->flags = NETPKT_F_VIEW; + return v; +} + void netpkt_ref(netpkt_t* p){ - if (p)p->refs++; + if (!p) return; + p->refs++; } void netpkt_unref(netpkt_t* p) { @@ -85,13 +499,47 @@ void netpkt_unref(netpkt_t* p) { p->refs--; return; } - if (p->free_fn) p->free_fn(p->free_ctx, p->base, p->alloc); - free_sized(p, sizeof(*p)); + + netpkt_buf_t* b = p->buf; + if (b) { + if (b->refs > 1) { + b->refs--; + } else { + if (b->flags & NETPKT_BUF_F_EXTERNAL) { + if (b->free_fn) b->free_fn(b->free_ctx, b->base, b->alloc); + } else { + bool aligned = ((b->base & (PAGE_SIZE - 1)) == 0) && ((b->alloc & (PAGE_SIZE - 1)) == 0); + if (!g_spare_page && aligned) { + g_spare_page = b->base; + if (b->alloc > PAGE_SIZE) { + pfree((void*)(b->base + PAGE_SIZE), (uint64_t)b->alloc-(uint64_t)PAGE_SIZE); + uint64_t dec = (uint64_t)b->alloc-(uint64_t)PAGE_SIZE; + if (g_netpkt_page_bytes >= dec) g_netpkt_page_bytes -= dec; + else g_netpkt_page_bytes = 0; + if (g_netpkt_payload_page_bytes >= dec) g_netpkt_payload_page_bytes -= dec; + else g_netpkt_payload_page_bytes = 0; + } + } else { + if (b->base) pfree((void*)b->base, (uint64_t)b->alloc); + uint64_t dec = (uint64_t)b->alloc; + if (g_netpkt_page_bytes >= dec) g_netpkt_page_bytes -= dec; + else g_netpkt_page_bytes = 0; + if (g_netpkt_payload_page_bytes >= dec) g_netpkt_payload_page_bytes -= dec; + else g_netpkt_payload_page_bytes = 0; + } + } + + meta_slab_free(&g_meta_slab_buf, b); + } + } + + meta_slab_free(&g_meta_slab_pkt, p); } uintptr_t netpkt_data(const netpkt_t* p) { if (!p) return 0; - return p->base + p->head; + if (!p->buf) return 0; + return p->buf->base+(uintptr_t)p->off+(uintptr_t)p->head; } uint32_t netpkt_len(const netpkt_t* p) { @@ -105,41 +553,70 @@ uint32_t netpkt_headroom(const netpkt_t* p) { uint32_t netpkt_tailroom(const netpkt_t* p) { if (!p) return 0; uint32_t used = p->head + p->len; - return used >= p->alloc ? 0: (p->alloc - used); + return used >= p->cap ? 0: (p->cap - used); } bool netpkt_ensure_headroom(netpkt_t* p, uint32_t need) { if (!p) return false; - if (p->head >= need) return true; + if (!p->buf) return false; + if (p->flags & NETPKT_F_VIEW) return false; + if (need > NETPKT_MAX_ALLOC) return false; + + if (p->head >= need) { + if (p->buf->refs == 1) return true; + return netpkt_realloc_to(p, p->head, p->cap); + } uint32_t tail = netpkt_tailroom(p); uint32_t new_head = need; - uint32_t new_alloc = new_head + p->len + tail; - if (new_alloc < p->alloc + (need - p->head)) new_alloc = p->alloc + (need - p->head); - return netpkt_realloc_to(p, new_head, new_alloc); + + uint64_t alloc = (uint64_t)new_head + (uint64_t)p->len + (uint64_t)tail; + uint64_t min = (uint64_t)p->cap + (uint64_t)(need - p->head); + if (alloc < min) alloc = min; + if (alloc > (uint64_t)NETPKT_MAX_ALLOC) return false; + + return netpkt_realloc_to(p, new_head, (uint32_t)alloc); } bool netpkt_ensure_tailroom(netpkt_t* p, uint32_t need) { if (!p) return false; - if (netpkt_tailroom(p) >= need) return true; + if (!p->buf) return false; + if (p->flags & NETPKT_F_VIEW) return false; + if (need > NETPKT_MAX_ALLOC) return false; - uint32_t new_alloc = p->head + p->len + need; - if (new_alloc < p->alloc + (need - netpkt_tailroom(p))) new_alloc = p->alloc + (need - netpkt_tailroom(p)); - return netpkt_realloc_to(p, p->head, new_alloc); + uint32_t tail = netpkt_tailroom(p); + if (tail >= need) { + if (p->buf->refs == 1) return true; + return netpkt_realloc_to(p, p->head, p->cap); + } + + uint64_t alloc = (uint64_t)p->head + (uint64_t)p->len + (uint64_t)need; + uint64_t min = (uint64_t)p->cap + (uint64_t)(need - tail); + if (alloc < min) alloc = min; + if (alloc > (uint64_t)NETPKT_MAX_ALLOC) return false; + + return netpkt_realloc_to(p, p->head, (uint32_t)alloc); } void* netpkt_push(netpkt_t* p, uint32_t bytes) { - if (!p || bytes == 0) return (void*)netpkt_data(p); + if (!p) return 0; + + if ((p->flags & NETPKT_F_VIEW) && bytes) return 0; + if (!bytes) return (void*)netpkt_data(p); if (!netpkt_ensure_headroom(p, bytes)) return 0; + p->head -= bytes; p->len += bytes; - return (void*)(p->base + p->head); + return (void*)(p->buf->base + (uintptr_t)p->off + (uintptr_t)p->head); } void* netpkt_put(netpkt_t* p, uint32_t bytes) { - if (!p || bytes == 0) return (void*)(netpkt_data(p) + p->len); + if (!p) return 0; + if ((p->flags & NETPKT_F_VIEW) && bytes) return 0; + if (!bytes) return (void*)(netpkt_data(p) + (uintptr_t)p->len); if (!netpkt_ensure_tailroom(p, bytes)) return 0; - uintptr_t out = p->base + p->head + p->len; + + uintptr_t out = p->buf->base+(uintptr_t)p->off + (uintptr_t)p->head + (uintptr_t)p->len; p->len += bytes; return (void*)out; } @@ -149,6 +626,31 @@ bool netpkt_pull(netpkt_t* p, uint32_t bytes) { if (bytes > p->len) return false; p->head += bytes; p->len -= bytes; + + if (p->flags & NETPKT_F_VIEW) return true; + if (!p->buf) return true; + if (p->buf->flags & NETPKT_BUF_F_EXTERNAL) return true; + if (p->buf->refs != 1) return true; + if (p->cap <= PAGE_SIZE) return true; + + uint64_t need = (uint64_t)p->head+(uint64_t)p->len; + if (!need) need = 1; + uint64_t newcap64 = count_pages(need, PAGE_SIZE) * (uint64_t)PAGE_SIZE; + if (newcap64 < PAGE_SIZE) newcap64 = PAGE_SIZE; + if (newcap64 >= (uint64_t)p->cap) return true; + + uint32_t newcap = (uint32_t)newcap64; + uint64_t dec = (uint64_t)p->buf->alloc - (uint64_t)newcap; + if (dec && ((p->buf->base & (PAGE_SIZE - 1)) == 0)) { + pfree((void*)(p->buf->base + (uintptr_t)newcap), dec); + if (g_netpkt_page_bytes >= dec) g_netpkt_page_bytes -= dec; + else g_netpkt_page_bytes = 0; + if (g_netpkt_payload_page_bytes >= dec) g_netpkt_payload_page_bytes -= dec; + else g_netpkt_payload_page_bytes = 0; + p->buf->alloc = newcap; + p->cap = newcap; + } + return true; } @@ -156,5 +658,30 @@ bool netpkt_trim(netpkt_t* p, uint32_t new_len) { if (!p) return false; if (new_len > p->len) return false; p->len = new_len; + + if (p->flags & NETPKT_F_VIEW) return true; + if (!p->buf) return true; + if (p->buf->flags & NETPKT_BUF_F_EXTERNAL) return true; + if (p->buf->refs != 1) return true; + if (p->cap <= PAGE_SIZE) return true; + + uint64_t need = (uint64_t)p->head + (uint64_t)p->len; + if (!need) need = 1; + uint64_t newcap64 = count_pages(need, PAGE_SIZE) * (uint64_t)PAGE_SIZE; + if (newcap64 < PAGE_SIZE) newcap64 = PAGE_SIZE; + if (newcap64 >= (uint64_t)p->cap) return true; + + uint32_t newcap = (uint32_t)newcap64; + uint64_t dec = (uint64_t)p->buf->alloc - (uint64_t)newcap; + if (dec && ((p->buf->base & (PAGE_SIZE - 1)) == 0)) { + pfree((void*)(p->buf->base + (uintptr_t)newcap), dec); + if (g_netpkt_page_bytes >= dec) g_netpkt_page_bytes -= dec; + else g_netpkt_page_bytes = 0; + if (g_netpkt_payload_page_bytes >= dec) g_netpkt_payload_page_bytes -= dec; + else g_netpkt_payload_page_bytes = 0; + p->buf->alloc = newcap; + p->cap = newcap; + } + return true; } diff --git a/kernel/networking/netpkt.h b/kernel/networking/netpkt.h index 441005cb..1f03c27f 100644 --- a/kernel/networking/netpkt.h +++ b/kernel/networking/netpkt.h @@ -10,8 +10,12 @@ typedef struct netpkt netpkt_t; typedef void (*netpkt_free_fn)(void* ctx, uintptr_t base, uint32_t alloc_size); +#define NETPKT_MAX_ALLOC 65536u +#define NETPKT_MAX_PAGE_BYTES (32ull * 1024ull * 1024ull) + netpkt_t* netpkt_alloc(uint32_t data_capacity, uint32_t headroom, uint32_t tailroom); -netpkt_t* netpkt_wrap(uintptr_t base, uint32_t alloc_size, uint32_t data_len, netpkt_free_fn free_fn, void* ctx); +netpkt_t* netpkt_wrap(uintptr_t base, uint32_t alloc_size, uint32_t data_off, uint32_t data_len, netpkt_free_fn free_fn, void* ctx); +netpkt_t* netpkt_view(netpkt_t* parent, uint32_t off, uint32_t len); void netpkt_ref(netpkt_t* p); void netpkt_unref(netpkt_t* p); diff --git a/kernel/networking/network_dispatch.cpp b/kernel/networking/network_dispatch.cpp index 1a3318fa..6dcb3efe 100644 --- a/kernel/networking/network_dispatch.cpp +++ b/kernel/networking/network_dispatch.cpp @@ -133,13 +133,12 @@ int NetworkDispatch::net_task() if (nics[n].rx.is_empty()) break; sizedptr pkt{0,0}; if (!nics[n].rx.pop(pkt)) break; - netpkt_t* np = netpkt_wrap(pkt.ptr, pkt.size, pkt.size, NULL, 0); + netpkt_t* np = netpkt_wrap(pkt.ptr, pkt.size, 0 , pkt.size, NULL, 0); if (np) { eth_input(nics[n].ifindex, np); netpkt_unref(np); - } else { - free_frame(pkt); } + free_frame(pkt); nics[n].rx_consumed++; processed++; } diff --git a/kernel/networking/processes/net_proc.c b/kernel/networking/processes/net_proc.c index b9d214e8..be81d6ae 100644 --- a/kernel/networking/processes/net_proc.c +++ b/kernel/networking/processes/net_proc.c @@ -37,7 +37,8 @@ #include "exceptions/timer.h" #include "syscalls/syscalls.h" - +#include "memory/page_allocator.h" +#include "memory/mmu.h" static int udp_probe_server(uint32_t probe_ip, uint16_t probe_port, net_l4_endpoint *out_l4) { socket_handle_t sock = udp_socket_create(SOCK_ROLE_CLIENT, (uint16_t)get_current_proc_pid(), NULL); @@ -80,7 +81,7 @@ static int udp_probe_server(uint32_t probe_ip, uint16_t probe_port, net_l4_endpo static void free_request(HTTPRequestMsg *req) { if (!req) return; - if (req->path.mem_length) free_sized(req->path.data, req->path.mem_length); + if (req->path.mem_length) string_free(req->path); http_headers_common_free(&req->headers_common); http_headers_extra_free(req->extra_headers, req->extra_header_count); @@ -95,6 +96,8 @@ static void free_request(HTTPRequestMsg *req) { static void run_http_server() { + //mmu_enable_verbose(); + //page_alloc_enable_verbose(); uint16_t pid = get_current_proc_pid(); SocketExtraOptions opt = {0}; opt.debug_level = SOCK_DBG_ALL; @@ -238,7 +241,7 @@ static void test_http(const net_l4_endpoint* ep) { http_headers_common_free(&resp.headers_common); - if (resp.reason.data && resp.reason.mem_length) free_sized(resp.reason.data, resp.reason.mem_length); + if (resp.reason.data && resp.reason.mem_length) string_free(resp.reason); http_headers_extra_free(resp.extra_headers, resp.extra_header_count); } diff --git a/kernel/networking/transport_layer/socket_udp.hpp b/kernel/networking/transport_layer/socket_udp.hpp index de064b3e..e295b5c1 100644 --- a/kernel/networking/transport_layer/socket_udp.hpp +++ b/kernel/networking/transport_layer/socket_udp.hpp @@ -116,10 +116,14 @@ class UDPSocket : public Socket { } static uint32_t dispatch(uint8_t ifindex, ip_version_t ipver, const void* src_ip_addr, const void* dst_ip_addr, uintptr_t frame_ptr, uint32_t frame_len, uint16_t src_port, uint16_t dst_port) { + UDPSocket* first = nullptr; for (UDPSocket* s = s_list_head; s; s = s->next) { - if (!socket_matches_dst(s, ifindex, ipver, dst_ip_addr, dst_port)) + if (!socket_matches_dst(s, ifindex, ipver, dst_ip_addr, dst_port)) continue; + if (!first) { + first = s; continue; + } uintptr_t copy = (uintptr_t)malloc(frame_len); if (!copy) continue; @@ -127,6 +131,11 @@ class UDPSocket : public Socket { memcpy((void*)copy, (const void*)frame_ptr, frame_len); s->on_receive(ipver, src_ip_addr, src_port, copy, frame_len); } + + if (first) { + first->on_receive(ipver, src_ip_addr, src_port, frame_ptr, frame_len); + return frame_len; + } if (frame_ptr && frame_len) free_sized((void*)frame_ptr, frame_len); return frame_len; } @@ -144,14 +153,6 @@ class UDPSocket : public Socket { free_sized((void*)ring[r_head].ptr, ring[r_head].size); r_head = (r_head + 1) % UDP_RING_CAP; } - uintptr_t copy = (uintptr_t)malloc(len); - if (!copy) { - if (ptr && len) free_sized((void*)ptr, len); - return; - } - - memcpy((void*)copy, (void*)ptr, len); - if (ptr && len) free_sized((void*)ptr, len); int nexti = (r_tail + 1) % UDP_RING_CAP; if (nexti == r_head) { @@ -160,7 +161,7 @@ class UDPSocket : public Socket { r_head = (r_head + 1) % UDP_RING_CAP; } - ring[r_tail].ptr = copy; + ring[r_tail].ptr = ptr; ring[r_tail].size = len; rx_bytes += len; diff --git a/kernel/pci/pci.c b/kernel/pci/pci.c old mode 100644 new mode 100755 index c1724df5..8b1278e3 --- a/kernel/pci/pci.c +++ b/kernel/pci/pci.c @@ -3,6 +3,8 @@ #include "exceptions/exception_handler.h" #include "memory/talloc.h" #include "memory/mmu.h" +#include "memory/addr.h" +#include "memory/va_layout.h" #include "std/memory_access.h" #include "hw/hw.h" #include "sysregs.h" @@ -126,8 +128,9 @@ uint64_t pci_make_addr(uint32_t bus, uint32_t slot, uint32_t func, uint32_t offs void find_pci(){ if (!initialized){ initialized = true; - for (uint64_t addr = PCI_BASE; addr < PCI_BASE + 0x10000000; addr += GRANULE_2MB) - register_device_memory_2mb(addr, addr); + paddr_t ecam_pa = (paddr_t)VIRT_TO_PHYS(PCI_BASE); + for (uint64_t off = 0; off < 0x10000000; off += GRANULE_2MB) + register_device_memory_2mb(PCI_BASE + off, (uint64_t)ecam_pa + off); } if (!g_pci_scanned){ @@ -259,7 +262,7 @@ uint64_t find_pci_device(uint32_t vendor_id, uint32_t device_id) { uint64_t vendor_device = read32(device_address); if ((vendor_device & 0xFFFF) == vendor_id && ((vendor_device >> 16) & 0xFFFF) == device_id) { - kprintf("[PCI] Found device at bus %i, slot %i, func %i -> %x", bus, slot, func, device_address); + kprintf("[PCI] Found device at bus %i, slot %i, func %i -> %llx", bus, slot, func, device_address); return device_address; }// else if (((vendor_device >> 16) & 0xFFFF) != 0xFFFF) @@ -275,7 +278,7 @@ void dump_pci_config(uint64_t base) { kprintf("[PCI] Dumping PCI Configuration Space:"); for (uint32_t offset = 0; offset < 0x40; offset += 4) { uint64_t val = read64(base + offset); - kprintf("Offset %x: %x",offset, val); + kprintf("Offset %x: %llx", offset, val); } } @@ -286,10 +289,11 @@ void pci_enable_device(uint64_t pci_addr){ } void pci_register(uint64_t mmio_addr, uint64_t mmio_size){ + if (kva_is_dmap((uintptr_t)mmio_addr)) mmio_addr = (uint64_t)dmap_kva_to_pa((uintptr_t)mmio_addr); uint64_t start = mmio_addr & ~(GRANULE_4KB - 1); uint64_t end = (mmio_addr + mmio_size + GRANULE_4KB - 1) & ~(GRANULE_4KB - 1); for (uint64_t addr = start; addr < end; addr += GRANULE_4KB) - register_device_memory(PHYS_TO_VIRT(addr), VIRT_TO_PHYS(addr)); + register_device_memory_dmap(mmio_pa_to_kva((paddr_t)addr)); } #pragma region Interrupts @@ -448,13 +452,15 @@ bool pci_setup_msix(uint64_t pci_addr, msix_irq_line* irq_lines, uint8_t line_si uint64_t bar_size = 0; pci_setup_bar(pci_addr, bir, &table_addr, &bar_size); pci_register(table_addr, bar_size); - kprintf("Setting up new bar for MSI-X %x + %x",table_addr, table_addr_offset); + kprintf("Setting up new bar for MSI-X %llx + %llx", table_addr, (uint64_t)table_addr_offset); } else { - kprintf("Bar %i setup at %x + %x",bir, table_addr, table_addr_offset); - pci_register(table_addr, GRANULE_4KB); + kprintf("Bar %i setup at %llx + %llx",bir, table_addr, (uint64_t)table_addr_offset); + uint64_t need = (uint64_t)table_addr_offset + ((uint64_t)line_size * sizeof(msix_table_entry)); + pci_register(table_addr, need); } - volatile msix_table_entry *msix_start = (volatile msix_table_entry *)(uintptr_t)(table_addr + table_addr_offset); + paddr_t table_pa = (paddr_t)(table_addr + (uint64_t)table_addr_offset); + volatile msix_table_entry *msix_start = (volatile msix_table_entry *)(uintptr_t)mmio_pa_to_kva(table_pa); for (uint32_t i = 0; i < line_size; ++i){ volatile msix_table_entry *msix_entry = msix_start + i; diff --git a/kernel/pci/pcie.c b/kernel/pci/pcie.c index c2d42f02..1f6d36de 100644 --- a/kernel/pci/pcie.c +++ b/kernel/pci/pcie.c @@ -47,7 +47,7 @@ int encode_ibar_size(u64 size) bool init_hostbridge(){ for (int i = 0; i < 10; i++){ - register_device_memory(PCI_BASE + (i * 0x1000), PCI_BASE + (i * 0x1000)); + register_device_memory_dmap(PCI_BASE + (i * 0x1000)); } uintptr_t base = VIRT_TO_PHYS(PCI_BASE); diff --git a/kernel/process/context_switch.S b/kernel/process/context_switch.S index a36e77db..d24c5d22 100644 --- a/kernel/process/context_switch.S +++ b/kernel/process/context_switch.S @@ -9,7 +9,8 @@ save_pc_interrupt: .global restore_context restore_context: - ldr x18, =cpec + adrp x18, cpec + add x18, x18, :lo12:cpec ldr x18, [x18] // Restore general-purpose registers ldp x0, x1, [x18, #(8 * 0)] @@ -51,45 +52,7 @@ restore_context: 3: ldr x17, [x18, #(8 * 32)] msr elr_el1, x17 - - mov x17, #0 - - ldr x18, =pttbr - - cmp x18, #0 - b.eq 4f - - ldr x18, [x18] - - cmp x18, #0 - b.eq 4f - - msr ttbr0_el1, x18 - - tlbi vmalle1is -4: dsb ish - isb - - mov x18, #0 + ldr x17, [x18, #(8 * 17)] + ldr x18, [x18, #(8 * 18)] eret - -.global mmu_swap -mmu_swap: - ldr x18, =pttbr - - cmp x18, #0 - b.eq 4f - - ldr x18, [x18] - - cmp x18, #0 - b.eq 4f - - msr ttbr0_el1, x18 - - tlbi vmalle1is -4: dsb ish - isb - - ret \ No newline at end of file diff --git a/kernel/process/kernel_syscall_impl.c b/kernel/process/kernel_syscall_impl.c index 5c80293e..ae3fb803 100644 --- a/kernel/process/kernel_syscall_impl.c +++ b/kernel/process/kernel_syscall_impl.c @@ -13,14 +13,20 @@ #include "filesystem/filesystem.h" #include "sysregs.h" #include "memory/mmu.h" +#include "memory/addr.h" +extern page_index *p_index; void* malloc(size_t size){ process_t* k = get_proc_by_pid(1); - uintptr_t heap_pa = mmu_translate(k->heap); - if (!heap_pa) return 0; - void* ptr = kalloc((void*)heap_pa, size, ALIGN_16B, MEM_PRIV_KERNEL); - if (size >= PAGE_SIZE) - register_allocation(k->alloc_map, ptr, size);//TODO: not fully correct, but this will become a syscall for pages soon so it won't matter + if (!k) return 0; + + int tr = 0; + paddr_t heap_pa = mmu_translate(0, k->heap, &tr); + if (tr) return 0; + + void* ptr = kalloc((void*)dmap_pa_to_kva(heap_pa), size, ALIGN_16B, MEM_PRIV_KERNEL); + if (ptr && size >= PAGE_SIZE && k->alloc_map) + register_allocation(k->alloc_map, ptr, size); return ptr; } @@ -29,15 +35,26 @@ void free_sized(void*ptr, size_t size){ } void* page_alloc(size_t size){ + if (!size) return 0; process_t* k = get_proc_by_pid(1);//TODO: can we make this more fragmented? This inside a syscall, current proc outside - void *ptr = palloc(size, MEM_PRIV_KERNEL, MEM_RW, true); - register_allocation(k->alloc_map, ptr, size); + void *ptr = palloc(size, MEM_PRIV_KERNEL, MEM_RW | MEM_NORM, true); + if (k && k->alloc_map && ptr) register_allocation(k->alloc_map, ptr, size); return ptr; } void page_free(void *ptr){ + if (!ptr) return; + if (((uintptr_t)ptr & (PAGE_SIZE - 1)) != 0) return; process_t* k = get_proc_by_pid(1);//TODO: can we make this more fragmented? This inside a syscall, current proc outside - free_registered(k->alloc_map, ptr); + if (k && k->alloc_map) { + free_registered(k->alloc_map, ptr); + return; + } + + size_t size = 0; + if (p_index) size = get_alloc_size(p_index, ptr); + if (!size) size = PAGE_SIZE; + pfree(ptr, size); } extern void printl(const char *str){ diff --git a/kernel/process/loading/elf_file.c b/kernel/process/loading/elf_file.c index 0d81d0ee..174c00f3 100644 --- a/kernel/process/loading/elf_file.c +++ b/kernel/process/loading/elf_file.c @@ -3,9 +3,11 @@ #include "process_loader.h" #include "dwarf.h" #include "memory/page_allocator.h" +#include "memory/talloc.h" #include "std/memory.h" #include "exceptions/irq.h" #include "sysregs.h" +#include "memory/addr.h" typedef struct elf_header { char magic[4];//should be " ELF" @@ -55,37 +57,63 @@ typedef struct elf_section_header } elf_section_header; void get_elf_debug_info(process_t* proc, void* file, size_t filesize){ + if (!proc) return; + if (!file) return; + if (filesize < sizeof(elf_header)) return; + elf_header *header = (elf_header*)file; if (header->magic[0] != 0x7f){ kprintf("Failed to read header file"); return; } + if (header->magic[1] != 'E') return; + if (header->magic[2] != 'L') return; + if (header->magic[3] != 'F') return; + + if (header->section_entry_size != sizeof(elf_section_header)) return; + if (header->section_num_entries == 0) return; + if (header->section_header_offset >= filesize) return; + + size_t sh_total = (size_t)header->section_num_entries * sizeof(elf_section_header); + if (header->section_header_offset + sh_total > filesize) return; + if (header->string_table_section_index >= header->section_num_entries) return; elf_section_header *sections = (elf_section_header*)(file + header->section_header_offset); + if (sections[header->string_table_section_index].sh_offset >= filesize) return; + if (sections[header->string_table_section_index].sh_offset + sections[header->string_table_section_index].sh_size > filesize) return; + sizedptr debug_line = {}; sizedptr debug_line_str = {}; for (int i = 1; i < header->section_num_entries; i++){ + if (sections[i].sh_name >= sections[header->string_table_section_index].sh_size) continue; + char *section_name = (char*)(file + sections[header->string_table_section_index].sh_offset + sections[i].sh_name); - if (strcmp_case(".debug_line", section_name,true) == 0){ - debug_line = (sizedptr){(uintptr_t)file + sections[i].sh_offset,sections[i].sh_size}; + if (!debug_line.ptr && strcmp_case(".debug_line", section_name,true) == 0){ + if (sections[i].sh_offset < filesize && sections[i].sh_offset + sections[i].sh_size <= filesize){ + debug_line = (sizedptr){(uintptr_t)file + sections[i].sh_offset,sections[i].sh_size}; + } } - if (strcmp_case(".debug_line_str", file + sections[header->string_table_section_index].sh_offset + sections[i].sh_name,true) == 0) { - debug_line_str = (sizedptr){(uintptr_t)file + sections[i].sh_offset,sections[i].sh_size}; + + if (!debug_line_str.ptr && strcmp_case(".debug_line_str", section_name,true) == 0){ + if (sections[i].sh_offset < filesize && sections[i].sh_offset + sections[i].sh_size <= filesize){ + debug_line_str = (sizedptr){(uintptr_t)file + sections[i].sh_offset,sections[i].sh_size}; + } } + if (debug_line.ptr && debug_line_str.ptr) break; } if (debug_line.ptr && debug_line.size){ proc->debug_lines.size = debug_line.size; - void* dl = palloc(debug_line.size, MEM_PRIV_SHARED, MEM_RO, true); + void* dl = palloc(debug_line.size, MEM_PRIV_KERNEL, MEM_RO, true); memcpy(dl, (void*)debug_line.ptr, debug_line.size); proc->debug_lines.ptr = (uintptr_t)dl; } if (debug_line_str.ptr && debug_line_str.size){ proc->debug_line_str.size = debug_line_str.size; - void* dls = palloc(debug_line_str.size, MEM_PRIV_SHARED, MEM_RO, true); + void* dls = palloc(debug_line_str.size, MEM_PRIV_KERNEL, MEM_RO, true); memcpy(dls, (void*)debug_line_str.ptr, debug_line_str.size); proc->debug_line_str.ptr = (uintptr_t)dls; } @@ -99,6 +127,9 @@ uint8_t elf_to_red_permissions(uint8_t flags){ } process_t* load_elf_file(const char *name, const char *bundle, void* file, size_t filesize){ + if (!file) return 0; + if (filesize < sizeof(elf_header)) return 0; + elf_header *header = (elf_header*)file; if (header->magic[0] != 0x7f){ @@ -106,6 +137,9 @@ process_t* load_elf_file(const char *name, const char *bundle, void* file, size_ return 0; } + if (header->magic[1] != 'E') return 0; + if (header->magic[2] != 'L') return 0; + if (header->magic[3] != 'F') return 0; // kprintf("ELF FILE VERSION %x HEADER VERSION %x (%x)",header->elf_version,header->header_version,header->header_size); // kprintf("There are %i program headers",header->program_header_num_entries); // kprintf("FILE %i for %x",header->type, header->instruction_set); @@ -119,38 +153,183 @@ process_t* load_elf_file(const char *name, const char *bundle, void* file, size_ // kprintf("Sections %i. String at %i. Offset %x",header->section_num_entries,header->string_table_section_index,header->section_header_offset); + if (header->program_header_entry_size != sizeof(elf_program_header)) return 0; + if (header->program_header_num_entries == 0) return 0; + if (header->program_header_offset >= filesize) return 0; + + size_t ph_total = (size_t)header->program_header_num_entries * sizeof(elf_program_header); + if (header->program_header_offset + ph_total > filesize) return 0; + elf_program_header* program_headers = (elf_program_header*)(file + header->program_header_offset); - - program_load_data data[header->program_header_num_entries] = {}; - + + bool use_sections = false; + for (int i = 0; i < header->program_header_num_entries; i++) { + if (program_headers[i].segment_type != 1) continue; + if (program_headers[i].p_memsz == 0) continue; + if ((elf_to_red_permissions(program_headers[i].flags) & (MEM_RW | MEM_EXEC)) == (MEM_RW | MEM_EXEC)){ + use_sections = true; + break; + } + } + + if (use_sections){ //should this fallback be kept or should it be removed? + kprintf("ELF has RWX PT_LOAD, using fallback"); + if (header->section_entry_size == sizeof(elf_section_header) && header->section_num_entries && header->section_header_offset < filesize) { + size_t sh_total = (size_t)header->section_num_entries * sizeof(elf_section_header); + if (header->section_header_offset + sh_total <= filesize) { + elf_section_header *sections = (elf_section_header*)(file + header->section_header_offset); + size_t sec_count = 0; + for (int i = 1; i < header->section_num_entries; i++) { + if (!(sections[i].sh_flags & 2)) continue; + if (!sections[i].sh_size) continue; + if (sections[i].sh_type != 8) { + if (sections[i].sh_offset >= filesize) continue; + if (sections[i].sh_offset + sections[i].sh_size > filesize) continue; + } + sec_count++; + } + if (sec_count) { + program_load_data *data = (program_load_data*)talloc(sec_count * sizeof(program_load_data)); + if (data){ + size_t di = 0; + for (int i = 1; i < header->section_num_entries; i++) { + if (!(sections[i].sh_flags & 2)) continue; + if (!sections[i].sh_size) continue; + if (sections[i].sh_type != 8) { + if (sections[i].sh_offset >= filesize) continue; + if (sections[i].sh_offset + sections[i].sh_size > filesize) continue; + } + + uint8_t perm = 0; + if (sections[i].sh_flags & 4) perm |= MEM_EXEC; + if (sections[i].sh_flags & 1) perm |= MEM_RW; + + sizedptr file_cpy = {}; + if (sections[i].sh_type != 8) { + file_cpy.ptr = (uintptr_t)file + sections[i].sh_offset; + file_cpy.size = sections[i].sh_size; + } + + data[di] = (program_load_data) { + .permissions = perm, + .file_cpy = file_cpy, + .virt_mem = (sizedptr){sections[i].sh_addr,sections[i].sh_size} + }; + di++; + } + + if (di) { + process_t *proc = create_process(name, bundle, data, di, header->program_entry_offset, true); + temp_free(data, sec_count * sizeof(program_load_data)); + if (!proc) return 0; + + proc->PROC_X0 = 1; + + size_t blen = strlen(bundle); + size_t nlen = strlen(name); + size_t plen = blen + nlen + 5; + uintptr_t sp = proc->stack - (plen+1); + paddr_t sp_phys = proc->stack_phys - (plen+1); + + char *nargvals = (char*)dmap_pa_to_kva(sp_phys); + + memcpy(nargvals, bundle, blen); + *(char*)(nargvals + blen) = '/'; + memcpy(nargvals + blen + 1, name, nlen); + memcpy(nargvals + blen+ 1+ nlen, ".elf", 4); + + *(char*)(nargvals + plen) = 0; + + uintptr_t pad = sp & 15; + sp -= pad; + sp_phys -= pad; + sp -= 2 * sizeof(uintptr_t); + sp_phys -= 2 * sizeof(uintptr_t); + *(uintptr_t*)dmap_pa_to_kva(sp_phys) = sp + 2 * sizeof(uintptr_t) + pad; + + *(uintptr_t*)dmap_pa_to_kva(sp_phys + sizeof(uintptr_t)) = 0; + + proc->PROC_X1 = sp; + proc->sp = sp; + + get_elf_debug_info(proc, file, filesize); + + return proc; + } + temp_free(data, sec_count * sizeof(program_load_data)); + } + } + } + } + } + size_t load_count = 0; + for (int i = 0; i < header->program_header_num_entries; i++) { + if (program_headers[i].segment_type != 1) continue; + if (program_headers[i].p_memsz == 0) continue; + if (program_headers[i].p_offset > filesize) continue; + if (program_headers[i].p_filez > filesize) continue; + if (program_headers[i].p_offset + program_headers[i].p_filez > filesize) continue; + load_count++; + } + if (load_count == 0) return 0; + + program_load_data* data = (program_load_data*)talloc(load_count * sizeof(program_load_data)); + if (!data) return 0; + + size_t di = 0; for (int i = 0; i < header->program_header_num_entries; i++){ // kprintf("Load to %llx (%llx + %llx) for %llx (%llx) %b at %x", program_headers[i].p_vaddr, program_headers[i].p_offset, program_headers[i].p_offset, program_headers[i].p_memsz, program_headers[i].p_filez, program_headers[i].flags, program_headers[i].alignment); - data[i] = (program_load_data){ + if (program_headers[i].segment_type != 1) continue; + if (program_headers[i].p_memsz == 0) continue; + if (program_headers[i].p_offset > filesize) continue; + if (program_headers[i].p_filez > filesize) continue; + if (program_headers[i].p_offset + program_headers[i].p_filez > filesize) continue; + + data[di] = (program_load_data){ .permissions = elf_to_red_permissions(program_headers[i].flags), .file_cpy = (sizedptr){(uintptr_t)file + program_headers[i].p_offset,program_headers[i].p_filez}, .virt_mem = (sizedptr){program_headers[i].p_vaddr,program_headers[i].p_memsz} }; + di++; } - process_t *proc = create_process(name, bundle, data, header->program_header_num_entries, header->program_entry_offset); + if (di == 0) { + temp_free(data, load_count * sizeof(program_load_data)); + return 0; + } + + process_t *proc = create_process(name, bundle, data, di, header->program_entry_offset, false); + temp_free(data, load_count * sizeof(program_load_data)); + if (!proc) return 0; proc->PROC_X0 = 1; - string s = string_format("%s/%s.elf",bundle,name); - - char *nargvals = (char*)(PHYS_TO_VIRT_P(proc->stack_phys)-s.length-1-sizeof(uintptr_t)); + size_t blen = strlen(bundle); + size_t nlen = strlen(name); + size_t plen = blen + nlen + 1 + 4; + uintptr_t sp = proc->stack - (plen+1); + paddr_t sp_phys = proc->stack_phys - (plen+1); - memcpy(nargvals, s.data, s.length); + char *nargvals = (char*)dmap_pa_to_kva(sp_phys); - *(char*)(nargvals+s.length) = 0; + memcpy(nargvals, bundle, blen); + *(char*)(nargvals+blen) = '/'; + memcpy(nargvals + blen + 1, name, nlen); + memcpy(nargvals + blen+ 1+ nlen, ".elf", 4); - *(uintptr_t*)PHYS_TO_VIRT_P(proc->stack_phys-sizeof(uintptr_t)) = (uintptr_t)nargvals; + *(char*)(nargvals+plen) = 0; - proc->PROC_X1 = proc->stack_phys-sizeof(uintptr_t); + uintptr_t pad = sp & 15; + sp -= pad; + sp_phys -= pad; + sp -= 2 * sizeof(uintptr_t); + sp_phys -= 2 * sizeof(uintptr_t); + *(uintptr_t*)dmap_pa_to_kva(sp_phys) = sp + 2 * sizeof(uintptr_t) + pad; - proc->sp -= s.length+1+sizeof(uintptr_t); + *(uintptr_t*)dmap_pa_to_kva(sp_phys + sizeof(uintptr_t)) = 0; - string_free(s); + proc->PROC_X1 = sp; + proc->sp = sp; get_elf_debug_info(proc, file, filesize); diff --git a/kernel/process/loading/process_loader.c b/kernel/process/loading/process_loader.c index 3f941663..e3902c6f 100644 --- a/kernel/process/loading/process_loader.c +++ b/kernel/process/loading/process_loader.c @@ -8,6 +8,7 @@ #include "memory/mmu.h" #include "memory/talloc.h" #include "sysregs.h" +#include "memory/addr.h" typedef struct { uint64_t code_base_start; @@ -48,19 +49,19 @@ uint32_t print_branch(uint32_t instr, uint64_t pc, bool translate, process_layou int32_t imm26 = instr & 0x03FFFFFF; int32_t signed_imm = (imm26 << 6) >> 6; uint64_t target = pc + ((int64_t)signed_imm << 2); - kputfv("%x /*pc = %x*/", target, pc); + kputfv("%llx /*pc = %llx*/", target, pc); if (!translate) return instr; bool internal = (target >= source->code_base_start) && (target < source->code_base_start + source->code_size); - uint32_t offset = pc - source->code_base_start; + uint64_t offset = pc - source->code_base_start; if (!internal) { uint64_t dest_pc = destination->code_base_start + offset; int64_t rel = (int64_t)(target - dest_pc) >> 2; if (rel < -(1 << 25) || rel >= (1 << 25)) { - kputfv("O.O.R. %x", rel);//We need to account for out of range + kputfv("O.O.R. %llx", rel);//We need to account for out of range } return (instr & 0xFC000000) | ((uint32_t)(rel & 0x03FFFFFF)); } @@ -78,7 +79,7 @@ uint32_t print_cond_branch(uint32_t instr, uint64_t pc, bool translate, process_ "eq","ne","cs","cc","mi","pl","vs","vc", "hi","ls","ge","lt","gt","le","","invalid" }; - kputfv("%x, %s", target, (uintptr_t)cond_names[cond]); + kputfv("%llx, %s", target, cond_names[cond]); if (!translate) return instr; @@ -88,7 +89,7 @@ uint32_t print_cond_branch(uint32_t instr, uint64_t pc, bool translate, process_ if (!internal) { int64_t rel = (int64_t)(target - (destination->code_base_start + new_offset)) >> 2; - instr = (instr & 0xFC000000) | (rel & 0x03FFFFFF); + instr = (instr & ~(0x7FFFFu << 5)) | ((uint32_t)(rel & 0x7FFFF) << 5); } return instr; } @@ -107,8 +108,10 @@ uint32_t print_adrp(uint32_t instr, uint64_t pc, bool translate, process_layout uint32_t rd = instr & 0x1F; uint64_t immhi = (instr >> 5) & 0x7FFFF; uint64_t immlo = (instr >> 29) & 0x3; - int64_t offset = ((int64_t)((immhi << 2) | immlo) << 44) >> 32; - kputfv("x%i, %x", rd, (pc & ~0xFFFUL) + offset); + int64_t imm21 = (int64_t)((immhi << 2) | immlo); + imm21 = (imm21 << 43) >> 43;//b + int64_t offset = imm21 << 12; + kputfv("x%i, %llx", rd, (pc & ~0xFFFUL) + offset); if (!translate) return instr; @@ -139,7 +142,7 @@ uint32_t print_adrp(uint32_t instr, uint64_t pc, bool translate, process_layout uint32_t print_ldr_str(uint32_t instr, uint64_t pc, bool translate, process_layout *source, process_layout *destination) { uint32_t rt = instr & 0x1F; uint32_t rn = (instr >> 5) & 0x1F; - uint32_t imm = ((instr >> 10) & 0xFFF) << 3; + uint32_t imm = ((instr >> 10) & 0xFFF) << (instr >> 30) & 0x3; if (rt == 31) kputfv("xzr, [x%i]", rn); else @@ -174,7 +177,7 @@ uint32_t print_mov32(uint32_t instr, uint64_t pc, bool translate, process_layout uint32_t print_movr(uint32_t instr, uint64_t pc, bool translate, process_layout *source, process_layout *destination) { uint32_t rd = instr & 0x1F; - uint32_t imm16 = (instr >> 15) & 0x1F; + uint32_t imm16 = (instr >> 16) & 0x1F; kputfv("x%i, x%i", rd, imm16); return instr; } @@ -209,7 +212,8 @@ void decode_instruction(uint32_t instruction){ kprintf("Instruction code %x",instruction); for (uint64_t i = 0; i < N_ARR(ops); i++) { if ((instruction & ops[i].mask) == ops[i].pattern) { - kputf("%s ", (uintptr_t)ops[i].mnemonic); + kputf("%s ", ops[i].mnemonic); + break; } } } @@ -217,8 +221,8 @@ void decode_instruction(uint32_t instruction){ uint32_t parse_instruction(uint32_t instruction, uint64_t pc, bool translate, process_layout *source, process_layout *destination){ for (uint64_t i = 0; i < N_ARR(ops); i++) { if ((instruction & ops[i].mask) == ops[i].pattern) { - kputfv("%s ", (uintptr_t)ops[i].mnemonic); - uint64_t newinstr = ops[i].reloc(instruction, pc, translate, source, destination); + kputfv("%s ", ops[i].mnemonic); + uint32_t newinstr = ops[i].reloc(instruction, pc, translate, source, destination); return newinstr; } } @@ -245,7 +249,7 @@ void relocate_code(void* dst, void* src, uint32_t size, uint64_t src_data_base, .data_size = data_size, }; - kprintfv("Beginning translation from base address %x to new address %x", src_base, dst_base); + kprintfv("Beginning translation from base address %llx to new address %llx", src_base, dst_base); for (uint32_t i = 0; i < count; i++) { uint32_t instr = src32[i]; @@ -261,13 +265,13 @@ void relocate_code(void* dst, void* src, uint32_t size, uint64_t src_data_base, kprintfv("Finished translation"); } -size_t map_section(process_t *proc, uintptr_t base, uintptr_t off, program_load_data data){ +size_t map_section(process_t *proc, kaddr_t base, uaddr_t off, program_load_data data){ // kprintf("Copying %llx from %llx to %llx, representing %llx",data.file_cpy.size,data.file_cpy.ptr,base + (data.virt_mem.ptr - off), data.virt_mem.size); - if (data.file_cpy.size) memcpy((void*)base + (data.virt_mem.ptr - off), (void*)data.file_cpy.ptr, data.file_cpy.size); + if (data.file_cpy.size) memcpy((void*)(uintptr_t)base + ((uintptr_t)data.virt_mem.ptr - (uintptr_t)off), (void*)data.file_cpy.ptr, data.file_cpy.size); return data.virt_mem.size; } -process_t* create_process(const char *name, const char *bundle, program_load_data *data, size_t data_count, uintptr_t entry) { +process_t* create_process(const char *name, const char *bundle, program_load_data *data, size_t data_count, uintptr_t entry, bool allow_rwx) { process_t* proc = init_process(); @@ -277,90 +281,149 @@ process_t* create_process(const char *name, const char *bundle, program_load_dat proc->alloc_map = make_page_index(); - uintptr_t min_addr = UINT64_MAX; - uintptr_t max_addr = 0; + uaddr_t min_addr = UINT64_MAX; + uaddr_t max_addr = 0; for (size_t i = 0; i < data_count; i++){ - if (data[i].virt_mem.ptr < min_addr) min_addr = data[i].virt_mem.ptr; - if (data[i].virt_mem.ptr + data[i].virt_mem.size > max_addr) max_addr = data[i].virt_mem.ptr + data[i].virt_mem.size; + uaddr_t s0 = (uaddr_t)data[i].virt_mem.ptr; + uaddr_t s1 = (uaddr_t)(data[i].virt_mem.ptr + data[i].virt_mem.size); + if (s0 < min_addr) min_addr = s0; + if (s1 > max_addr) max_addr = s1; } + uaddr_t min_map = min_addr & ~(GRANULE_4KB - 1); + uaddr_t max_map = (max_addr + (GRANULE_4KB - 1)) & ~(GRANULE_4KB - 1); - size_t code_size = max_addr-min_addr; + size_t code_size = max_map -min_map; // kprintf("Code takes %x from %x to %x",code_size, min_addr, max_addr); uintptr_t *ttbr = mmu_new_ttbr(); - uintptr_t *kttbr = mmu_default_ttbr(); - uintptr_t dest = (uintptr_t)palloc_inner(code_size, MEM_PRIV_USER, MEM_EXEC | MEM_RW, true, false); - register_allocation(proc->alloc_map, (void*)dest, code_size); - if (!dest) return 0; + memset(&proc->mm, 0, sizeof(proc->mm)); + proc->mm.ttbr0 = ttbr; + proc->mm.ttbr0_phys = pt_va_to_pa(ttbr); + + paddr_t dest = palloc_inner(code_size, MEM_PRIV_USER, MEM_RW, true, false); + if (!dest) { + reset_process(proc); + return 0; + } // kprintf("Allocated space for process between %x and %x",dest,dest+((code_size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1))); - for (uintptr_t i = min_addr; i < max_addr; i += GRANULE_4KB){ - mmu_map_4kb(kttbr, (uintptr_t)dest + (i - min_addr), (uintptr_t)dest + (i - min_addr), MAIR_IDX_NORMAL, MEM_EXEC | MEM_RO | MEM_NORM, MEM_PRIV_USER); - mmu_map_4kb(ttbr, i, (uintptr_t)dest + (i - min_addr), MAIR_IDX_NORMAL, MEM_EXEC | MEM_RO | MEM_NORM, MEM_PRIV_USER); - } - memset(PHYS_TO_VIRT_P(dest), 0, code_size); - proc->use_va = true; - - for (size_t i = 0; i < data_count; i++) - map_section(proc, PHYS_TO_VIRT(dest), min_addr, data[i]); + for (uaddr_t va = min_map; va < max_map; va += GRANULE_4KB) { + bool any = 0; + bool rw = 0; + bool ex = 0; + + for (size_t s = 0; s < data_count; s++){ + uaddr_t s0 = (uaddr_t)data[s].virt_mem.ptr; + uaddr_t s1 = (uaddr_t)(data[s].virt_mem.ptr + data[s].virt_mem.size); + if (va + GRANULE_4KB <= s0) continue; + if (va >= s1) continue; + any = true; + if (data[s].permissions & MEM_RW) rw = true; + if (data[s].permissions & MEM_EXEC) ex = true; + } - proc->va = min_addr; - proc->code = (void*)dest; - proc->code_size = (code_size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + if (!any) continue; + if (rw && ex && !allow_rwx) { + //kprintf("WX overlap at page %llx", va); + if (dest) pfree((void*)dmap_pa_to_kva(dest), code_size); + reset_process(proc); + return 0; + } + if (rw && !allow_rwx) ex = false; - max_addr = (max_addr + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); - proc->last_va_mapping = max_addr; - - uint64_t stack_size = 0x10000; + uint8_t attr = MEM_NORM; + if (rw) attr |= MEM_RW; + if (ex) attr |= MEM_EXEC; + mmu_map_4kb((uint64_t*)ttbr, (uint64_t)va, (paddr_t)(dest + (va - min_map)), MAIR_IDX_NORMAL, attr, MEM_PRIV_USER); + } + memset((void*)dmap_pa_to_kva(dest), 0, code_size); - uintptr_t stack = (uintptr_t)palloc_inner(stack_size, MEM_PRIV_USER, MEM_RW, true, false); - register_allocation(proc->alloc_map, (void*)stack, stack_size); - if (!stack) return 0; - - proc->last_va_mapping += PAGE_SIZE;//Unmapped page to catch stack overflows - proc->stack = (proc->last_va_mapping + stack_size); - proc->stack_phys = (stack + stack_size); - - for (uintptr_t i = stack; i < stack + stack_size; i += GRANULE_4KB){ - mmu_map_4kb(kttbr, i, i, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_USER); - mmu_map_4kb(ttbr, proc->last_va_mapping, i, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_USER); - mmu_map_4kb(ttbr, i, i, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_USER); - proc->last_va_mapping += PAGE_SIZE; + for (size_t i = 0; i < data_count; i++) { + uint8_t prot = MEM_NORM; + if (data[i].permissions & MEM_RW) prot |= MEM_RW; + if (data[i].permissions & MEM_EXEC) prot |= MEM_EXEC; + mm_add_vma(&proc->mm, data[i].virt_mem.ptr, data[i].virt_mem.ptr + data[i].virt_mem.size, prot, VMA_KIND_ELF, 0); + } + for (size_t i = 0; i < data_count; i++) + map_section(proc, dmap_pa_to_kva(dest), min_map, data[i]); + + proc->va = min_map; + proc->code = dest; + proc->code_size = code_size; + + uint64_t stack_commit_size = 0x10000; + uint64_t stack_max_size = 0x800000; //TODO it shouldnt be fix + uaddr_t stack_top = 0x00007FFFFFFFF000ULL; + uaddr_t stack_limit = stack_top - stack_max_size; + uaddr_t stack_commit = stack_top - stack_commit_size; + uaddr_t mmap_top = stack_limit - PAGE_SIZE; + + uaddr_t heap_start = (max_map + (PAGE_SIZE*4) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + if (heap_start >= mmap_top) { + reset_process(proc); + return 0; + } + proc->heap = heap_start; + proc->heap_phys = 0; + + proc->mm.heap_start = heap_start; + proc->mm.brk = heap_start; + proc->mm.mmap_top = mmap_top; + proc->mm.mmap_cursor = mmap_top; + proc->mm.brk_max = mmap_top - (PAGE_SIZE * MM_GAP_PAGES); + proc->mm.stack_top = stack_top; + proc->mm.stack_limit = stack_limit; + proc->mm.stack_commit = stack_commit; + + if (proc->mm.brk_max < proc->mm.brk) { + reset_process(proc); + return 0; } - memset(PHYS_TO_VIRT_P(stack), 0, stack_size); - proc->last_va_mapping += PAGE_SIZE;//Unmapped page to catch stack overflows + uint64_t total_pages = get_total_user_ram() / PAGE_SIZE; + if (!total_pages) total_pages = 1; - uint8_t heapattr = MEM_RW; + proc->mm.cap_stack_pages = stack_max_size / PAGE_SIZE; + proc->mm.cap_heap_pages = (proc->mm.brk_max - proc->mm.heap_start) / PAGE_SIZE; + proc->mm.cap_anon_pages = total_pages / 2; + if (proc->mm.cap_anon_pages < 128) proc->mm.cap_anon_pages = 128; - uintptr_t heap = (uintptr_t)palloc_inner(PAGE_SIZE, MEM_PRIV_USER, MEM_RW, false, false); - register_allocation(proc->alloc_map, (void*)heap, PAGE_SIZE); - if (!heap) return 0; + mm_add_vma(&proc->mm, proc->mm.stack_limit, proc->mm.stack_top, MEM_RW, VMA_KIND_STACK, VMA_FLAG_DEMAND); - proc->heap = proc->last_va_mapping; - proc->heap_phys = heap; - mmu_map_4kb(ttbr, proc->last_va_mapping, heap, MAIR_IDX_NORMAL, heapattr | MEM_NORM, MEM_PRIV_USER); - mmu_map_4kb(ttbr, heap, heap, MAIR_IDX_NORMAL, heapattr | MEM_NORM, MEM_PRIV_USER); - mmu_map_4kb(kttbr, heap, heap, MAIR_IDX_NORMAL, heapattr | MEM_NORM, MEM_PRIV_USER); + paddr_t stack = palloc_inner(stack_commit_size, MEM_PRIV_USER, MEM_RW, true, false); + if (!stack) { + reset_process(proc); + return 0; + } - setup_page(PHYS_TO_VIRT(heap), heapattr); + proc->stack = stack_top; + proc->stack_phys = stack + stack_commit_size; - memset(PHYS_TO_VIRT_P(heap + sizeof(mem_page)), 0, PAGE_SIZE - sizeof(mem_page)); + for (paddr_t pa = stack; pa < stack + stack_commit_size; pa += GRANULE_4KB) { + uaddr_t va = stack_commit + (uaddr_t)(pa - stack); + mmu_map_4kb((uint64_t*)ttbr, (uint64_t)va, pa, MAIR_IDX_NORMAL, MEM_RW | MEM_NORM, MEM_PRIV_USER); + } + memset((void*)dmap_pa_to_kva(stack), 0, stack_commit_size); - proc->last_va_mapping += PAGE_SIZE; + proc->stack_size = stack_commit_size; + proc->mm.rss_stack_pages = stack_commit_size / PAGE_SIZE; - proc->stack_size = stack_size; + proc->heap = proc->mm.brk; + proc->sp = proc->stack; - proc->ttbr = ttbr; + proc->output = (kaddr_t)palloc(PROC_OUT_BUF, MEM_PRIV_KERNEL, MEM_RW, true); + if (!proc->output) { + reset_process(proc); + return 0; + } + proc->output_size = 0; - proc->sp = proc->stack; - proc->pc = (uintptr_t)(entry); - kprintf("User process %s allocated with address at %llx, stack at %llx-%llx (%llx-%llx), heap at %llx (%llx)",(uintptr_t)name,proc->pc, proc->sp - proc->stack_size, proc->sp, proc->stack_phys - proc->stack_size, proc->stack_phys, proc->heap, proc->heap_phys); + kprintf("User process %s pc=%llx stack=%llx-%llx (phys=%llx-%llx) heap=%llx (phys=%llx)", name, (uint64_t)proc->pc, (uint64_t)(proc->sp - proc->stack_size), (uint64_t)proc->sp, (uint64_t)(proc->stack_phys - proc->stack_size), (uint64_t)proc->stack_phys, (uint64_t)proc->heap, (uint64_t)proc->heap_phys); proc->spsr = 0; proc->state = BLOCKED; diff --git a/kernel/process/loading/process_loader.h b/kernel/process/loading/process_loader.h index 9d3b3626..a7902f02 100644 --- a/kernel/process/loading/process_loader.h +++ b/kernel/process/loading/process_loader.h @@ -12,7 +12,7 @@ typedef struct { uint8_t permissions; } program_load_data; -process_t* create_process(const char *name, const char *bundle, program_load_data *data, size_t data_count, uintptr_t entry); +process_t* create_process(const char *name, const char *bundle, program_load_data *data, size_t data_count, uintptr_t entry, bool allow_rwx); void translate_enable_verbose(); void decode_instruction(uint32_t instruction); #ifdef __cplusplus diff --git a/kernel/process/process.h b/kernel/process/process.h index ec638004..80011fce 100644 --- a/kernel/process/process.h +++ b/kernel/process/process.h @@ -8,7 +8,7 @@ extern "C" { #include "keyboard_input.h" #include "net/network_types.h" #include "dev/driver_base.h" -#include "memory/memory_types.h" +#include "memory/mm_process.h" #define INPUT_BUFFER_CAPACITY 64 #define PACKET_BUFFER_CAPACITY 128 @@ -34,7 +34,7 @@ typedef struct { #define MAX_PROC_NAME_LENGTH 256 -typedef struct { +typedef struct process { //We use the addresses of these variables to save and restore process state uint64_t regs[31]; // x0–x30 uintptr_t sp; @@ -43,21 +43,18 @@ typedef struct { //Not used in process saving uint16_t id; uintptr_t stack; - uintptr_t stack_phys; + paddr_t stack_phys; uint64_t stack_size; uintptr_t heap; - uintptr_t heap_phys; - uintptr_t output; + paddr_t heap_phys; + kaddr_t output; size_t output_size; - uintptr_t last_va_mapping; file out_fd; int32_t exit_code; bool focused; - void *code; + paddr_t code; size_t code_size; - bool use_va; - uintptr_t *ttbr; - uintptr_t va; + uaddr_t va; page_index *alloc_map; enum process_state { STOPPED, READY, RUNNING, BLOCKED } state; __attribute__((aligned(16))) input_buffer_t input_buffer; @@ -65,11 +62,15 @@ typedef struct { __attribute__((aligned(16))) packet_buffer_t packet_buffer; uint8_t priority; uint16_t win_id; + uaddr_t win_fb_va; + paddr_t win_fb_phys; + uint64_t win_fb_size; char *bundle; char name[MAX_PROC_NAME_LENGTH]; sizedptr debug_lines; sizedptr debug_line_str; system_module exposed_fs; + mm_struct mm; } process_t; //Helper functions for accessing registers mapped to scratch regs diff --git a/kernel/process/scheduler.c b/kernel/process/scheduler.c index acb56b70..b7441c19 100644 --- a/kernel/process/scheduler.c +++ b/kernel/process/scheduler.c @@ -11,6 +11,7 @@ #include "math/math.h" #include "memory/mmu.h" #include "process/syscall.h" +#include "memory/addr.h" #include "sysregs.h" #include "filesystem/filesystem.h" #include "dev/module_loader.h" @@ -42,10 +43,10 @@ void save_return_address_interrupt(){ save_pc_interrupt(cpec); } -extern void mmu_swap(); void switch_proc(ProcSwitchReason reason) { // kprintf("Stopping execution of process %i at %x",current_proc, processes[current_proc].spsr); + if (mmu_ttbr0_user_enabled()) panic("switch_proc with user ttbr0 active", current_proc); if (proc_count == 0) panic("No processes active", 0); int next_proc = (current_proc + 1) % MAX_PROCS; @@ -56,7 +57,8 @@ void switch_proc(ProcSwitchReason reason) { current_proc = next_proc; cpec = (uintptr_t)&processes[current_proc]; timer_reset(processes[current_proc].priority); - mmu_swap_ttbr(processes[current_proc].ttbr); + if (processes[current_proc].mm.ttbr0) mmu_asid_ensure(&processes[current_proc].mm); + mmu_swap_ttbr(processes[current_proc].mm.ttbr0 ? &processes[current_proc].mm : 0); process_restore(); } @@ -65,6 +67,10 @@ void save_syscall_return(uint64_t value){ } void process_restore(){ + if ((processes[current_proc].spsr & 0xF) == 0) { + if (!processes[current_proc].mm.ttbr0) panic("process_restore user process without ttbr0", processes[current_proc].id); + mmu_ttbr0_enable_user(); + } else mmu_ttbr0_disable_user(); restore_context(cpec); } @@ -92,6 +98,7 @@ bool init_scheduler_module(){ uintptr_t get_current_heap(){ + if (processes[current_proc].mm.ttbr0) return processes[current_proc].mm.brk; return processes[current_proc].heap; } @@ -115,17 +122,19 @@ uint16_t get_current_proc_pid(){ } void reset_process(process_t *proc){ - bool just_finished = processes[current_proc].id == proc->id; + uint16_t pid = proc->id; + int32_t exit_code = proc->exit_code; + uint8_t state = proc->state; + bool counted = state != STOPPED; + + bool just_finished = processes[current_proc].id == pid; proc->sp = 0; proc->pc = 0; proc->spsr = 0; - proc->exit_code = 0; - if (!just_finished && proc->output) - kfree((void*)proc->output, PROC_OUT_BUF); for (int j = 0; j < 31; j++) proc->regs[j] = 0; - for (int k = 0; k < MAX_PROC_NAME_LENGTH; k++) - proc->name[k] = 0; + //for (int k = 0; k < MAX_PROC_NAME_LENGTH; k++) + //proc->name[k] = 0; proc->input_buffer.read_index = 0; proc->input_buffer.write_index = 0; for (int k = 0; k < INPUT_BUFFER_CAPACITY; k++){ @@ -139,20 +148,82 @@ void reset_process(process_t *proc){ free_sizedptr(p); proc->packet_buffer.entries[k] = (sizedptr){0}; } - if (!just_finished && proc->alloc_map) release_page_index(proc->alloc_map); - close_files_for_process(proc->id); - if (proc->ttbr) { - if (pttbr == proc->ttbr) { - kprintf("[PROC error] Trying to free process while mapped", (uintptr_t)proc->ttbr); - return; + close_files_for_process(pid); + + if (proc->debug_lines.ptr) { + pfree((void*)proc->debug_lines.ptr, proc->debug_lines.size); + proc->debug_lines = (sizedptr){0}; + } + if (proc->debug_line_str.ptr) { + pfree((void*)proc->debug_line_str.ptr, proc->debug_line_str.size); + proc->debug_line_str = (sizedptr){0}; + } + + if (proc->output) { + pfree((void*)proc->output, PROC_OUT_BUF); + proc->output = 0; + proc->output_size = 0; + } + + if (proc->mm.ttbr0) { + for (uint16_t i = 0; i < proc->mm.vma_count; i++) { + vma *m = &proc->mm.vmas[i]; + bool nofree = (m->flags & VMA_FLAG_NOFREE) != 0; + for (uaddr_t va = m->start; va < m->end; va += GRANULE_4KB) { + paddr_t pa = 0; + if (!mmu_unmap_and_get_pa((uint64_t*)proc->mm.ttbr0, (uint64_t)va, &pa)) continue; + if (!nofree) pfree((void*)dmap_pa_to_kva(pa), GRANULE_4KB); + if (m->kind == VMA_KIND_HEAP) { + if (proc->mm.rss_heap_pages) proc->mm.rss_heap_pages--; + } else if (m->kind == VMA_KIND_STACK) { + if (proc->mm.rss_stack_pages) proc->mm.rss_stack_pages--; + } else if (m->kind == VMA_KIND_ANON) { + if (proc->mm.rss_anon_pages) proc->mm.rss_anon_pages--; + } + } + } + proc->mm.vma_count = 0; + } + + if (proc->alloc_map) { + if (!proc->mm.ttbr0) { + for (page_index *ind = proc->alloc_map; ind; ind = ind->header.next) ind->header.size = 0; } - mmu_free_ttbr(proc->ttbr); + release_page_index(proc->alloc_map); + proc->alloc_map = 0; + } + if (proc->mm.ttbr0) { + mmu_asid_release(&proc->mm); + mmu_free_ttbr(proc->mm.ttbr0); + proc->mm.ttbr0 = 0; + proc->mm.ttbr0_phys = 0; } if (proc->exposed_fs.init){ unload_module(&proc->exposed_fs); } - if (!just_finished) + proc->exposed_fs = (system_module){0}; + + for (int k = 0; k < MAX_PROC_NAME_LENGTH; k++) proc->name[k] = 0; + + proc->stack = 0; + proc->stack_phys = 0; + proc->stack_size = 0; + + proc->heap = 0; + proc->heap_phys = 0; + memset(&proc->mm, 0, sizeof(proc->mm)); + + proc->win_id = 0; + if (counted && proc_count) proc_count--; + + if (!just_finished) { memset(proc, 0, sizeof(process_t)); + return; + } + + proc->id = pid; + proc->exit_code = exit_code; + proc->state = state; } void init_main_process(){ @@ -163,10 +234,16 @@ void init_main_process(){ proc->alloc_map = make_page_index(); proc->state = BLOCKED; proc->heap = (uintptr_t)palloc(0x1000, MEM_PRIV_KERNEL, MEM_RW, false); + proc->heap_phys = (paddr_t)VIRT_TO_PHYS(proc->heap); proc->stack_size = 0x10000; proc->stack = (uintptr_t)palloc(proc->stack_size,MEM_PRIV_KERNEL, MEM_RW,true); - proc->sp = ksp; + proc->sp = (uintptr_t)ksp; + proc->output = (kaddr_t)palloc(PROC_OUT_BUF, MEM_PRIV_KERNEL, MEM_RW, true); + proc->output_size = 0; proc->priority = PROC_PRIORITY_LOW; + proc->win_fb_va = 0; + proc->win_fb_phys = 0; + proc->win_fb_size = 0; name_process(proc, "kernel"); proc_count++; } @@ -178,8 +255,13 @@ process_t* init_process(){ if (processes[i].state == STOPPED){ proc = &processes[i]; reset_process(proc); + for (int k = 0; k < MAX_PROC_NAME_LENGTH; k++) proc->name[k] = 0; proc->state = READY; proc->id = next_proc_index++; + proc->priority = PROC_PRIORITY_LOW; + proc->win_fb_va = 0; + proc->win_fb_phys = 0; + proc->win_fb_size = 0; proc_count++; return proc; } @@ -189,9 +271,13 @@ process_t* init_process(){ proc = &processes[next_proc_index]; reset_process(proc); + for (int k = 0; k < MAX_PROC_NAME_LENGTH; k++) proc->name[k] = 0; proc->id = next_proc_index++; proc->state = READY; proc->priority = PROC_PRIORITY_LOW; + proc->win_fb_va = 0; + proc->win_fb_phys = 0; + proc->win_fb_size = 0; proc_count++; return proc; } @@ -206,18 +292,23 @@ void name_process(process_t *proc, const char *name){ void stop_process(uint16_t pid, int32_t exit_code){ disable_interrupt(); process_t *proc = get_proc_by_pid(pid); - if (proc->state != READY) return; + if (!proc || proc->state != READY) { + enable_interrupt(); + return; + } + + bool current = proc == &processes[current_proc]; proc->state = STOPPED; proc->exit_code = exit_code; if (proc->focused) sys_unset_focus(); - if (proc->ttbr) { - mmu_swap_ttbr(0); - mmu_swap(); - } + if (current && proc->mm.ttbr0) mmu_swap_ttbr(0); reset_process(proc); - proc_count--; // kprintf("Stopped %i process %i",pid,proc_count); + if (!current) { + enable_interrupt(); + return; + } switch_proc(HALT); } @@ -277,14 +368,14 @@ void wake_processes(){ bool load_process_module(process_t *p, system_module *m){ p->exposed_fs = *m; - p->exposed_fs.init = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.init - p->va)); - p->exposed_fs.fini = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.fini - p->va)); - p->exposed_fs.open = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.open - p->va)); - p->exposed_fs.read = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.read - p->va)); - p->exposed_fs.write = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.write - p->va)); - p->exposed_fs.close = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.close - p->va)); - p->exposed_fs.sread = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.sread - p->va)); - p->exposed_fs.swrite = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.swrite - p->va)); + p->exposed_fs.init = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.init - (uintptr_t)p->va)); + p->exposed_fs.fini = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.fini - (uintptr_t)p->va)); + p->exposed_fs.open = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.open - (uintptr_t)p->va)); + p->exposed_fs.read = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.read - (uintptr_t)p->va)); + p->exposed_fs.write = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.write - (uintptr_t)p->va)); + p->exposed_fs.close = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.close - (uintptr_t)p->va)); + p->exposed_fs.sread = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.sread - (uintptr_t)p->va)); + p->exposed_fs.swrite = PHYS_TO_VIRT_P(p->code + ((uintptr_t)p->exposed_fs.swrite - (uintptr_t)p->va)); return load_module(&p->exposed_fs); } @@ -334,17 +425,27 @@ FS_RESULT open_proc(const char *path, file *descriptor){ path = seek_to(pid_s, '/'); uint64_t pid = parse_int_u64(pid_s, path - pid_s); process_t *proc = get_proc_by_pid(pid); + if (!proc) return FS_RESULT_NOTFOUND; descriptor->id = fid; descriptor->cursor = 0; module_file *file = kalloc(proc_page, sizeof(module_file), ALIGN_64B, MEM_PRIV_KERNEL); + if (!file) return FS_RESULT_DRIVER_ERROR; + memset(file, 0, sizeof(module_file)); file->fid = fid; + file->references = 1; if (strcmp_case(path, "out",true) == 0){ - if (!proc->output) proc->output = (uintptr_t)palloc(PROC_OUT_BUF, MEM_PRIV_KERNEL, MEM_RW, true); + if (!proc->output) proc->output = (kaddr_t)palloc(PROC_OUT_BUF, MEM_PRIV_KERNEL, MEM_RW, true); + if (!proc->output) { + kfree(file, sizeof(module_file)); + return FS_RESULT_DRIVER_ERROR; + } descriptor->size = proc->output_size; file->file_buffer = (buffer){ .buffer = (char*)proc->output, + .buffer_size = proc->output_size, .limit = PROC_OUT_BUF, - .options = buffer_circular + .options = buffer_circular, + .cursor = proc->output_size, }; } else if (strcmp_case(path, "state",true) == 0){ descriptor->size = sizeof(int); @@ -354,8 +455,12 @@ FS_RESULT open_proc(const char *path, file *descriptor){ .limit = sizeof(int), .options = buffer_static, .buffer_size = sizeof(int), + .cursor = 0, }; - } else return FS_RESULT_NOTFOUND; + } else { + kfree(file, sizeof(module_file)); + return FS_RESULT_NOTFOUND; + } file->file_size = descriptor->size; return chashmap_put(proc_opened_files, &descriptor->id, sizeof(uint64_t), file) >= 0 ? FS_RESULT_SUCCESS : FS_RESULT_DRIVER_ERROR; } @@ -382,6 +487,7 @@ size_t read_proc(file* fd, char *buf, size_t size, file_offset offset){ module_file *file = (module_file*)chashmap_get(proc_opened_files, &fd->id, sizeof(uint64_t)); if (!file) return 0; size_t s = buffer_read(&file->file_buffer, buf, size, offset); + fd->size = file->file_size; return s; } @@ -396,21 +502,35 @@ size_t write_proc(file* fd, const char *buf, size_t size, file_offset offset){ return 0; } module_file *file = (module_file*)chashmap_get(proc_opened_files, &fd->id, sizeof(uint64_t)); + if (!file) return 0; if (file->read_only) return 0; - bool is_output = (uintptr_t)file->file_buffer.buffer == get_current_proc()->output; + bool is_output = (uintptr_t)file->file_buffer.buffer == (uintptr_t)get_current_proc()->output; - size = min(size+1, file->file_buffer.limit); + size = min(size, file->file_buffer.limit); if (is_output){//TODO: probably better to make these files be held by this module, and created only when needed - fd->cursor = file->file_size; + size_t written= buffer_write_lim(&file->file_buffer, buf, size); - fd->cursor += buffer_write_lim(&file->file_buffer, buf, size); - fd->cursor += buffer_write(&file->file_buffer, "\n"); - - file->file_size += size; - - get_current_proc()->output_size += size; + file->file_size = file->file_buffer.buffer_size; + get_current_proc()->output_size = file->file_size; + fd->size = file->file_size; + return written; + } + return 0; +} + +void close_proc(file *fd) { + if (!fd) return; + if (!proc_opened_files) return; + + uint64_t fid = fd->id; + module_file *mfile = (module_file*)chashmap_get(proc_opened_files, &fid, sizeof(fid)); + if (!mfile) return; + + if (mfile->references > 0) mfile->references--; + if (mfile->references == 0) { + chashmap_remove(proc_opened_files, &fid, sizeof(fid), 0); + kfree(mfile, sizeof(module_file)); } - return size; } system_module scheduler_module = (system_module){ @@ -422,7 +542,7 @@ system_module scheduler_module = (system_module){ .open = open_proc, .read = read_proc, .write = write_proc, - .close = 0, + .close = close_proc, .sread = 0, .swrite = 0,//TODO implement simple io .readdir = list_processes, diff --git a/kernel/process/scheduler.h b/kernel/process/scheduler.h index 3a1d0242..54513cf7 100644 --- a/kernel/process/scheduler.h +++ b/kernel/process/scheduler.h @@ -26,6 +26,7 @@ void process_restore(); void stop_process(uint16_t pid, int32_t exit_code); void stop_current_process(int32_t exit_code); +void reset_process(process_t *proc); void name_process(process_t *proc, const char *name); @@ -53,4 +54,4 @@ process_t *get_all_processes(); extern system_module scheduler_module; -extern uint64_t ksp; +extern char ksp[]; diff --git a/kernel/process/syscall.c b/kernel/process/syscall.c index 4c69b957..3a5baa6d 100644 --- a/kernel/process/syscall.c +++ b/kernel/process/syscall.c @@ -7,6 +7,7 @@ #include "process/process.h" #include "process/scheduler.h" #include "memory/page_allocator.h" +#include "memory/talloc.h" #include "graph/graphics.h" #include "std/memory_access.h" #include "input/input_dispatch.h" @@ -25,8 +26,12 @@ #include "networking/transport_layer/csocket.h" #include "loading/dwarf.h" #include "sysregs.h" +#include "memory/addr.h" #include "ui/graphic_types.h" #include "dev/module_loader.h" +#include "alloc/page_index.h" +#include "process/uaccess.h" +#include "syscalls/errno.h" int syscall_depth = 0; uintptr_t cpec; @@ -35,56 +40,178 @@ uintptr_t cpec; typedef uint64_t (*syscall_entry)(process_t *ctx); u64 syscall_malloc(process_t *ctx){ - void* page_ptr = (void*)mmu_translate(syscall_depth > 1 ? get_proc_by_pid(1)->heap : ctx->heap); - if ((uintptr_t)page_ptr == 0x0){ - handle_exception("Wrong process heap state", 0); + if (syscall_depth > 1) { + process_t *k = get_proc_by_pid(1); + if (!k) return 0; + return (u64)kalloc((void*)dmap_pa_to_kva((paddr_t)k->heap_phys), ctx->PROC_X0, ALIGN_16B, MEM_PRIV_KERNEL); } + size_t size = ctx->PROC_X0; - void *ptr = kalloc_inner(page_ptr, size, ALIGN_16B, get_current_privilege(), ctx->heap, &ctx->last_va_mapping, ctx->ttbr); - if (size >= PAGE_SIZE) - register_allocation(ctx->alloc_map, ptr, size);//TODO: not fully correct, but this will become a syscall for pages soon so it won't matter - return (uintptr_t)ptr; + if (!size) return 0; + + u64 pages = count_pages(size, PAGE_SIZE); + size_t alloc_size = pages * PAGE_SIZE; + if (ctx->mm.rss_anon_pages + pages > ctx->mm.cap_anon_pages) return 0; + + uptr va = mm_alloc_mmap(&ctx->mm, alloc_size, MEM_RW, VMA_KIND_ANON, VMA_FLAG_DEMAND | VMA_FLAG_USERALLOC | VMA_FLAG_ZERO); + if (!va) return 0; + + mmu_flush_asid(ctx->mm.asid); + return va; } uptr syscall_palloc(process_t *ctx){ size_t size = ctx->PROC_X0; - void *ptr = palloc(size, MEM_PRIV_USER, MEM_RW, true); - register_allocation(ctx->alloc_map, ptr, size); - if (ctx->use_va){ - uptr va = ctx->last_va_mapping; - u64 pages = count_pages(size, PAGE_SIZE); - for (u64 i = 0; i < pages; i++){ - mmu_map_4kb(ctx->ttbr, va + (i * PAGE_SIZE), (uptr)ptr + (i * PAGE_SIZE), MAIR_IDX_NORMAL, MEM_RW, MEM_PRIV_USER); - } - ptr = (void*)ctx->last_va_mapping; - ctx->last_va_mapping += (pages * PAGE_SIZE); + if(!size) return 0; + u64 pages = count_pages(size, PAGE_SIZE); + size_t alloc_size = pages * PAGE_SIZE; + + if (ctx->mm.ttbr0){ + if (ctx->mm.rss_anon_pages + pages > ctx->mm.cap_anon_pages) return 0; + //TODO zalloc likely needs a dedicated syscall to request zeroed on fault explicitly + uptr va = mm_alloc_mmap(&ctx->mm, alloc_size, MEM_RW, VMA_KIND_ANON, VMA_FLAG_DEMAND | VMA_FLAG_USERALLOC | VMA_FLAG_ZERO); + if (!va) return 0; + mmu_flush_asid(ctx->mm.asid); + return va; } - return (uptr)ptr; + + paddr_t ptr = palloc_inner(alloc_size, MEM_PRIV_USER, MEM_RW, true, true); + if(!ptr) return 0; + void *kva = (void*)dmap_pa_to_kva(ptr); + register_allocation(ctx->alloc_map, kva, alloc_size); + return (uptr)kva; } u64 syscall_pfree(process_t *ctx){ - free_registered(ctx->alloc_map, (void*)ctx->PROC_X0); + uptr va = ctx->PROC_X0; + if (!va) return SYSCALL_ERRNO(SYSCALL_EINVAL); + + if (ctx->mm.ttbr0) { + vma *m = mm_find_vma(&ctx->mm,va); + if (!m) return SYSCALL_ERRNO(SYSCALL_EFAULT); + if (m->kind != VMA_KIND_ANON) return SYSCALL_ERRNO(SYSCALL_EPERM); + if (!(m->flags & VMA_FLAG_USERALLOC)) return SYSCALL_ERRNO(SYSCALL_EPERM); + + uintptr_t start = m->start; + uintptr_t end = m->end; + if (!mm_remove_vma(&ctx->mm, start, end)) return SYSCALL_ERRNO(SYSCALL_EINVAL); + + for (uintptr_t a = start; a < end; a += PAGE_SIZE) { + uint64_t pa = 0; + if (!mmu_unmap_and_get_pa((uint64_t*)ctx->mm.ttbr0, a, &pa)) continue; + pfree((void*)dmap_pa_to_kva((paddr_t)pa), PAGE_SIZE); + if (ctx->mm.rss_anon_pages) ctx->mm.rss_anon_pages--; + } + + mmu_flush_asid(ctx->mm.asid); + return 0; + } + + size_t size = get_alloc_size(ctx->alloc_map, (void*)va); + if (!size) return SYSCALL_ERRNO(SYSCALL_EFAULT); + u64 pages = count_pages(size, PAGE_SIZE); + free_registered(ctx->alloc_map, (void*)va); + if (ctx->mm.rss_anon_pages >= pages) ctx->mm.rss_anon_pages -= pages; + else ctx->mm.rss_anon_pages = 0; return 0; + return SYSCALL_ERRNO(SYSCALL_EINVAL); } u64 syscall_free(process_t *ctx){ - kfree((void*)ctx->PROC_X0, ctx->PROC_X1); + if (ctx->mm.ttbr0) return syscall_pfree(ctx); + + void *ptr = (void*)ctx->PROC_X0; + size_t size = (size_t)ctx->PROC_X1; + if (!ptr || !size) return SYSCALL_ERRNO(SYSCALL_EINVAL); + if (!ctx->alloc_map) return SYSCALL_ERRNO(SYSCALL_EPERM); + + size_t alloc_size = get_alloc_size(ctx->alloc_map, ptr); + if (!alloc_size) return SYSCALL_ERRNO(SYSCALL_EFAULT); + if (size && alloc_size != size) return SYSCALL_ERRNO(SYSCALL_EINVAL); + + free_registered(ctx->alloc_map, ptr); return 0; } +uptr syscall_brk(process_t *ctx) { + uptr req = ctx->PROC_X0; + uptr old = ctx->mm.brk; + if (!req) return old; + + uptr new_brk = (req + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + uptr min_brk = ctx->mm.heap_start; + if (new_brk < min_brk) new_brk = min_brk; + + if (new_brk > ctx->mm.brk_max) return old; + + uptr mmap_guard = ctx->mm.mmap_cursor - (MM_GAP_PAGES*PAGE_SIZE); + if (ctx->mm.mmap_free_count) { + for (uint16_t i = 0; i < ctx->mm.mmap_free_count; i++) if (ctx->mm.mmap_free[i].start >= ctx->mm.brk && ctx->mm.mmap_free[i].start < mmap_guard) mmap_guard = ctx->mm.mmap_free[i].start - (MM_GAP_PAGES * PAGE_SIZE); + } + if (new_brk > mmap_guard)return old; + + if (new_brk < old) { + for (uptr va = new_brk; va < old; va += PAGE_SIZE) { + uint64_t pa = 0; + if (!mmu_unmap_and_get_pa((uint64_t*)ctx->mm.ttbr0, va, &pa)) continue; + pfree((void*)dmap_pa_to_kva((paddr_t)pa), PAGE_SIZE); + if (ctx->mm.rss_heap_pages) ctx->mm.rss_heap_pages--; + } + } + + if (new_brk != old) { + if (old == ctx->mm.heap_start && new_brk > ctx->mm.heap_start) mm_add_vma(&ctx->mm, ctx->mm.heap_start, new_brk, MEM_RW, VMA_KIND_HEAP, VMA_FLAG_DEMAND); + else if (new_brk == ctx->mm.heap_start) mm_remove_vma(&ctx->mm, ctx->mm.heap_start, old); + else { + vma *heap = mm_find_vma(&ctx->mm, ctx->mm.heap_start); + if (heap && heap->kind == VMA_KIND_HEAP) heap->end = new_brk; + } + ctx->mm.brk = new_brk; + ctx->heap = new_brk; + mmu_flush_asid(ctx->mm.asid); + } + + return ctx->mm.brk; +} + u64 syscall_printl(process_t *ctx){ - kprint((const char *)ctx->PROC_X0); + uintptr_t u = (uintptr_t)ctx->PROC_X0; + if (!u) return SYSCALL_ERRNO(SYSCALL_EINVAL); + + char buf[256] = {}; + + for (;;) { + size_t copied = 0; + bool term = false; + uaccess_result_t ur = copy_str_from_user(ctx, buf, sizeof(buf), u, &copied, &term); + if (ur != UACCESS_OK && ur != UACCESS_ENAMETOOLONG) return ur; + kprint(buf); + if (term) break; + if (!copied) break; + u += copied; + } + return 0; } u64 syscall_read_key(process_t *ctx){ - keypress *kp = (keypress*)ctx->PROC_X0; - return sys_read_input_current(kp); + uintptr_t up = (uintptr_t)ctx->PROC_X0; + keypress tmp = {}; + u64 r = sys_read_input_current(&tmp); + if (!r) return 0; + uaccess_result_t ur = copy_to_user(ctx, up, &tmp, sizeof(tmp)); + if (ur != UACCESS_OK) return ur; + return r; } u64 syscall_read_event(process_t *ctx){ - kbd_event *ev = (kbd_event*)ctx->PROC_X0; - return sys_read_event_current(ev); + uintptr_t up = (uintptr_t)ctx->PROC_X0; + kbd_event tmp = {}; + u64 r = sys_read_event_current(&tmp); + if (!r) return 0; + uaccess_result_t ur = copy_to_user(ctx, up, &tmp, sizeof(tmp)); + if (ur != UACCESS_OK) return ur; + return r; } u64 syscall_read_shortcut(process_t *ctx){ @@ -94,32 +221,58 @@ u64 syscall_read_shortcut(process_t *ctx){ u64 syscall_get_mouse(process_t *ctx){ //TEST: are we preventing the mouse from being read outside of window? - if (get_current_proc_pid() != ctx->id) return 0; - mouse_data *inp = (mouse_data*)ctx->PROC_X0; - inp->raw = get_raw_mouse_in(); - inp->position = convert_mouse_position(get_mouse_pos()); + if (get_current_proc_pid() != ctx->id) return SYSCALL_ERRNO(SYSCALL_EPERM); + uintptr_t up = (uintptr_t)ctx->PROC_X0; + mouse_data tmp = {}; + tmp.raw = get_raw_mouse_in(); + tmp.position = convert_mouse_position(get_mouse_pos()); + uaccess_result_t ur = copy_to_user(ctx, up, &tmp, sizeof(tmp)); + if (ur != UACCESS_OK) return ur; return 0; } uptr syscall_gpu_request_ctx(process_t *ctx){ - draw_ctx* d_ctx = (draw_ctx*)ctx->PROC_X0; - get_window_ctx(d_ctx); + uintptr_t up = (uintptr_t)ctx->PROC_X0; + draw_ctx tmp = {}; + get_window_ctx(&tmp); + uaccess_result_t ur = copy_to_user(ctx, up, &tmp, sizeof(tmp)); + if (ur != UACCESS_OK) return ur; return 0; } u64 syscall_gpu_flush(process_t *ctx){ - draw_ctx* d_ctx = (draw_ctx*)ctx->PROC_X0; - commit_frame(d_ctx,0 ); + uintptr_t up = (uintptr_t)ctx->PROC_X0; + draw_ctx tmp = {}; + uaccess_result_t ur = copy_from_user(ctx, &tmp, up, sizeof(tmp)); + if (ur != UACCESS_OK) return ur; + + draw_ctx win = {}; + get_window_ctx(&win); + if (!tmp.full_redraw) { + if (tmp.dirty_count > MAX_DIRTY_RECTS) return SYSCALL_ERRNO(SYSCALL_EINVAL); + if (!win.width || !win.height) return SYSCALL_ERRNO(SYSCALL_EINVAL); + for (uint32_t i = 0; i < tmp.dirty_count; i++) { + gpu_rect r = tmp.dirty_rects[i]; + if (r.point.x >= win.width || r.point.y >= win.height) return SYSCALL_ERRNO(SYSCALL_EINVAL); + if (r.size.width > win.width - r.point.x) return SYSCALL_ERRNO(SYSCALL_EINVAL); + if (r.size.height > win.height - r.point.y) return SYSCALL_ERRNO(SYSCALL_EINVAL); + } + } + + commit_frame(&tmp, 0); gpu_flush(); return 0; } u64 syscall_gpu_resize_ctx(process_t *ctx){ - draw_ctx *d_ctx = (draw_ctx*)ctx->PROC_X0; + uintptr_t up = (uintptr_t)ctx->PROC_X0; uint32_t width = (uint32_t)ctx->PROC_X1; uint32_t height = (uint32_t)ctx->PROC_X2; resize_window(width, height); - get_window_ctx(d_ctx); + draw_ctx tmp = {}; + get_window_ctx(&tmp); + uaccess_result_t ur = copy_to_user(ctx, up, &tmp, sizeof(tmp)); + if (ur != UACCESS_OK) return ur; gpu_flush(); return 0; } @@ -142,11 +295,68 @@ u64 syscall_halt(process_t *ctx){ } u64 syscall_exec(process_t *ctx){ - const char *prog_name = (const char*)ctx->PROC_X0; - int argc = ctx->PROC_X1; - const char **argv = (const char**)ctx->PROC_X2; + uintptr_t upath = (uintptr_t)ctx->PROC_X0; + int argc = (int)ctx->PROC_X1; + uintptr_t uargv = (uintptr_t)ctx->PROC_X2; + + if (argc < 0) return SYSCALL_ERRNO(SYSCALL_EINVAL); + + char prog_name[256] = {}; + size_t copied = 0; + bool term = false; + uaccess_result_t ur = copy_str_from_user(ctx, prog_name, sizeof(prog_name), upath, &copied, &term); + if (ur != UACCESS_OK) return ur; + if (!term) return SYSCALL_ERRNO(SYSCALL_ENAMETOOLONG); + + const int max_args = 64; + if (argc > max_args) return SYSCALL_ERRNO(SYSCALL_EINVAL); + + const char *argv[max_args + 1] = {}; + char *bufs[max_args] = {}; + uint64_t bufsz[max_args] = {}; + + for (int i = 0; i < argc; i++) { + uintptr_t up = 0; + ur = copy_from_user(ctx, &up, uargv + ((uintptr_t)i * sizeof(uintptr_t)), sizeof(up)); + if (ur != UACCESS_OK) break; + if (!up) { + ur = UACCESS_EINVAL; + break; + } + + char tmp[256] = {}; + size_t n = 0; + bool t = false; + ur = copy_str_from_user(ctx, tmp, sizeof(tmp), up, &n, &t); + if (ur != UACCESS_OK) break; + + if (!t) { + ur = UACCESS_ENAMETOOLONG; + break; + } + + uint64_t alloc = (n + 0xFFF) & ~0xFFFULL; + char *k = (char*)talloc(alloc); + if (!k) { + ur = UACCESS_ENOMEM; + break; + } + memcpy(k, tmp, n); + bufs[i] = k; + bufsz[i] = alloc; + argv[i] = k; + } + + if (ur != UACCESS_OK) { + for (int i = 0; i < argc; i++) if (bufs[i]) temp_free(bufs[i], bufsz[i]); + return ur; + } + process_t *p = execute(prog_name, argc, argv); if (p) p->win_id = ctx->win_id; + + for (int i = 0; i < argc; i++) if (bufs[i]) temp_free(bufs[i], bufsz[i]); + return p ? p->id : 0; } @@ -157,112 +367,362 @@ u64 syscall_get_time(process_t *ctx){ u64 syscall_socket_create(process_t *ctx){ Socket_Role role = (Socket_Role)ctx->PROC_X0; protocol_t protocol = (protocol_t)ctx->PROC_X1; - const SocketExtraOptions* extra = (const SocketExtraOptions*)ctx->PROC_X2; - SocketHandle *out_handle = (SocketHandle*)ctx->PROC_X3; - return create_socket(role, protocol, extra, ctx->id, out_handle); + uintptr_t uextra = (uintptr_t)ctx->PROC_X2; + uintptr_t uout = (uintptr_t)ctx->PROC_X3; + SocketExtraOptions extra = {}; + SocketExtraOptions *pe = 0; + if (uextra) { + uaccess_result_t ur = copy_from_user(ctx, &extra, uextra, sizeof(extra)); + if (ur != UACCESS_OK) return ur; + pe = &extra; + } + + SocketHandle out = {}; + i64 r = create_socket(role, protocol, pe, ctx->id, &out); + if (r < 0) return r; + + uaccess_result_t ur = copy_to_user(ctx, uout, &out, sizeof(out)); + if (ur != UACCESS_OK) return ur; + return r; } u64 syscall_socket_bind(process_t *ctx){ - SocketHandle *handle = (SocketHandle*)ctx->PROC_X0; + uintptr_t uhandle = (uintptr_t)ctx->PROC_X0; ip_version_t ip_version = (ip_version_t)ctx->PROC_X1; uint16_t port = (uint16_t)ctx->PROC_X2; - return bind_socket(handle, port, ip_version, ctx->id); + SocketHandle handle = {}; + uaccess_result_t ur = copy_from_user(ctx, &handle, uhandle, sizeof(handle)); + if (ur != UACCESS_OK) return ur; + return bind_socket(&handle, port, ip_version, ctx->id); } u64 syscall_socket_connect(process_t *ctx){ - SocketHandle *handle = (SocketHandle*)ctx->PROC_X0; + uintptr_t uhandle = (uintptr_t)ctx->PROC_X0; uint8_t dst_kind = (uint8_t)ctx->PROC_X1; - void* dst = (void*)ctx->PROC_X2; + uintptr_t udst = (uintptr_t)ctx->PROC_X2; uint16_t port = (uint16_t)ctx->PROC_X3; - return connect_socket(handle, dst_kind, dst, port, ctx->id); + + SocketHandle handle = {}; + uaccess_result_t ur = copy_from_user(ctx, &handle, uhandle, sizeof(handle)); + if (ur != UACCESS_OK) return ur; + + net_l4_endpoint ep = {}; + char domain[256] = {}; + const void *dst = 0; + + if (dst_kind == DST_ENDPOINT) { + ur = copy_from_user(ctx, &ep, udst, sizeof(ep)); + if (ur != UACCESS_OK) return ur; + dst = &ep; + } else if (dst_kind == DST_DOMAIN) { + size_t copied = 0; + bool term = false; + ur = copy_str_from_user(ctx, domain, sizeof(domain), udst, &copied, &term); + if (ur != UACCESS_OK) return ur; + if (!term) return SYSCALL_ERRNO(SYSCALL_ENAMETOOLONG); + dst = domain; + } else { + return SYSCALL_ERRNO(SYSCALL_EINVAL); + } + + return connect_socket(&handle, dst_kind, dst, port, ctx->id); } u64 syscall_socket_listen(process_t *ctx){ - SocketHandle *handle = (SocketHandle*)ctx->PROC_X0; + uintptr_t uhandle = (uintptr_t)ctx->PROC_X0; int32_t backlog = (int32_t)ctx->PROC_X1; - return listen_on(handle, backlog, ctx->id); + + SocketHandle handle = {}; + uaccess_result_t ur = copy_from_user(ctx, &handle, uhandle, sizeof(handle)); + if (ur != UACCESS_OK) return ur; + return listen_on(&handle, backlog, ctx->id); } u64 syscall_socket_accept(process_t *ctx){ - SocketHandle *handle = (SocketHandle*)ctx->PROC_X0; - accept_on_socket(handle, ctx->id); + uintptr_t uhandle = (uintptr_t)ctx->PROC_X0; + SocketHandle handle = {}; + uaccess_result_t ur = copy_from_user(ctx, &handle, uhandle, sizeof(handle)); + if (ur != UACCESS_OK) return ur; + accept_on_socket(&handle, ctx->id); return 1; } u64 syscall_socket_send(process_t *ctx){ - SocketHandle *handle = (SocketHandle*)ctx->PROC_X0; + uintptr_t uhandle = (uintptr_t)ctx->PROC_X0; uint8_t dst_kind = (uint8_t)ctx->PROC_X1; - void* dst = (void*)ctx->PROC_X2; + uintptr_t udst = (uintptr_t)ctx->PROC_X2; uint16_t port = (uint16_t)ctx->PROC_X3; - void *ptr = (void*)ctx->PROC_X4; + uintptr_t ubuf = (uintptr_t)ctx->PROC_X4; size_t size = (size_t)ctx->regs[5]; - return send_on_socket(handle, dst_kind, dst, port, ptr, size, ctx->id); + + SocketHandle handle = {}; + uaccess_result_t ur = copy_from_user(ctx, &handle, uhandle, sizeof(handle)); + if (ur != UACCESS_OK) return ur; + + net_l4_endpoint ep = {}; + char domain[256] = {}; + const void *dst = 0; + + if (dst_kind == DST_ENDPOINT){ + ur = copy_from_user(ctx, &ep, udst, sizeof(ep)); + if (ur != UACCESS_OK) return ur; + dst = &ep; + } else if (dst_kind == DST_DOMAIN){ + size_t copied = 0; + bool term = false; + ur = copy_str_from_user(ctx, domain, sizeof(domain), udst, &copied, &term); + if (ur != UACCESS_OK) return ur; + if (!term) return SYSCALL_ERRNO(SYSCALL_ENAMETOOLONG); + dst = domain; + } else { + return SYSCALL_ERRNO(SYSCALL_EINVAL); + } + + if (!size) return 0; + + uint64_t alloc_size = (size + 0xFFF) & ~0xFFFULL; + void *kbuf = (void*)talloc(alloc_size); + if (!kbuf) return SYSCALL_ERRNO(SYSCALL_ENOMEM); + + ur = copy_from_user(ctx, kbuf, ubuf, size); + if (ur != UACCESS_OK){ + temp_free(kbuf, alloc_size); + return ur; + } + + u64 r = send_on_socket(&handle, dst_kind, dst, port, kbuf, size, ctx->id); + temp_free(kbuf, alloc_size); + return r; } u64 syscall_socket_receive(process_t *ctx){ - SocketHandle *handle = (SocketHandle*)ctx->PROC_X0; - void* buf = (void*)ctx->PROC_X1; + uintptr_t uhandle = (uintptr_t)ctx->PROC_X0; + uintptr_t ubuf = (uintptr_t)ctx->PROC_X1; size_t size = (size_t)ctx->PROC_X2; - net_l4_endpoint* out_src = (net_l4_endpoint*)ctx->PROC_X3; - return receive_from_socket(handle, buf, size, out_src, ctx->id); + uintptr_t uout_src = (uintptr_t)ctx->PROC_X3; + + SocketHandle handle = {}; + uaccess_result_t ur = copy_from_user(ctx, &handle, uhandle, sizeof(handle)); + if (ur != UACCESS_OK) return ur; + if (!size) return 0; + + uint64_t alloc_size = (size + 0xFFF) & ~0xFFFULL; + void *kbuf = (void*)talloc(alloc_size); + if (!kbuf) return SYSCALL_ERRNO(SYSCALL_ENOMEM); + + net_l4_endpoint src = {}; + i64 r = receive_from_socket(&handle, kbuf, size, &src, ctx->id); + if (r > 0) { + ur = copy_to_user(ctx, ubuf, kbuf, (size_t)r); + if (ur != UACCESS_OK) r = ur; + } + if (r >= 0) { + ur = copy_to_user(ctx, uout_src, &src, sizeof(src)); + if (ur != UACCESS_OK) r = ur; + } + + temp_free(kbuf, alloc_size); + return r; } u64 syscall_socket_close(process_t *ctx){ - SocketHandle *handle = (SocketHandle*)ctx->PROC_X0; - return close_socket(handle, ctx->id); + uintptr_t uhandle = (uintptr_t)ctx->PROC_X0; + SocketHandle handle = {}; + uaccess_result_t ur = copy_from_user(ctx, &handle, uhandle, sizeof(handle)); + if (ur != UACCESS_OK) return ur; + return close_socket(&handle, ctx->id); } u64 syscall_openf(process_t *ctx){ - char *req_path = (char *)ctx->PROC_X0; + uintptr_t upath = (uintptr_t)ctx->PROC_X0; + uintptr_t udesc = (uintptr_t)ctx->PROC_X1; + + char req_path[255] = {}; + size_t copied = 0; + bool term = false; + uaccess_result_t ur = copy_str_from_user(ctx, req_path, sizeof(req_path), upath, &copied, &term); + if (ur != UACCESS_OK) return ur; + if (!term) return SYSCALL_ERRNO(SYSCALL_ENAMETOOLONG); + char path[255] = {}; - if (!(ctx->PROC_PRIV) && strstart_case("/resources/", req_path,true) == 11){ - string_format_buf(path, sizeof(path),"%s%s", ctx->bundle, req_path); - } else memcpy(path, req_path, strlen(req_path) + 1); - //TODO: Restrict access to own bundle, own fs and require privilege escalation for full-ish filesystem access - file *descriptor = (file*)ctx->PROC_X1; - return open_file(path, descriptor); + if (!(ctx->PROC_PRIV) && strstart_case("/resources/", req_path, true) == 11){ + string_format_buf(path, sizeof(path), "%s%s", ctx->bundle, req_path); + } else { + memcpy(path, req_path, strlen(req_path) + 1); + } + file descriptor = {}; + FS_RESULT r = open_file(path, &descriptor); + if (r != FS_RESULT_SUCCESS) return r; + ur = copy_to_user(ctx, udesc, &descriptor, sizeof(descriptor)); + if (ur != UACCESS_OK) return ur; + return r; } u64 syscall_readf(process_t *ctx){ - file *descriptor = (file*)ctx->PROC_X0; - char *buf = (char*)ctx->PROC_X1; + uintptr_t udesc = (uintptr_t)ctx->PROC_X0; + uintptr_t ubuf = (uintptr_t)ctx->PROC_X1; size_t size = (size_t)ctx->PROC_X2; - return read_file(descriptor, buf, size); + file descriptor = {}; + uaccess_result_t ur = copy_from_user(ctx, &descriptor, udesc, sizeof(descriptor)); + if (ur != UACCESS_OK) return ur; + + uint8_t tmp[4096]; + size_t done = 0; + + while (done < size){ + size_t chunk = size - done; + if (chunk > sizeof(tmp)) chunk = sizeof(tmp); + + size_t r = read_file(&descriptor, (char*)tmp, chunk); + if (!r) break; + + ur = copy_to_user(ctx, ubuf + (uintptr_t)done, tmp, r); + if (ur != UACCESS_OK) { + if (!done) done = ur; + break; + } + + done += r; + if (r < chunk) break; + } + + ur = copy_to_user(ctx, udesc, &descriptor, sizeof(descriptor)); + if (ur != UACCESS_OK) return ur; + return done; } u64 syscall_writef(process_t *ctx){ - file *descriptor = (file*)ctx->PROC_X0; - char *buf = (char*)ctx->PROC_X1; + uintptr_t udesc = (uintptr_t)ctx->PROC_X0; + uintptr_t ubuf = (uintptr_t)ctx->PROC_X1; size_t size = (size_t)ctx->PROC_X2; - return write_file(descriptor, buf, size); + + file descriptor = {}; + uaccess_result_t ur = copy_from_user(ctx, &descriptor, udesc, sizeof(descriptor)); + if (ur != UACCESS_OK) return ur; + + uint8_t tmp[4096]; + size_t done = 0; + + while (done < size) { + size_t chunk = size - done; + if (chunk > sizeof(tmp)) chunk = sizeof(tmp); + + ur = copy_from_user(ctx, tmp, ubuf + (uintptr_t)done, chunk); + if (ur != UACCESS_OK) { + if (!done) done = ur; + break; + } + + size_t w = write_file(&descriptor, (const char*)tmp, chunk); + if (!w) break; + + done += w; + if (w < chunk) break; + } + + ur = copy_to_user(ctx, udesc, &descriptor, sizeof(descriptor)); + if (ur != UACCESS_OK) return ur; + return done; } u64 syscall_sreadf(process_t *ctx){ - const char *path = (const char*)ctx->PROC_X0; - void *buf = (void*)ctx->PROC_X1; + uintptr_t upath = (uintptr_t)ctx->PROC_X0; + uintptr_t ubuf = (uintptr_t)ctx->PROC_X1; size_t size = (size_t)ctx->PROC_X2; - return simple_read(path, buf, size); + + char path[255] = {}; + size_t copied = 0; + bool term = false; + uaccess_result_t ur = copy_str_from_user(ctx, path, sizeof(path), upath, &copied, &term); + if (ur != UACCESS_OK) return ur; + if (!term) return SYSCALL_ERRNO(SYSCALL_ENAMETOOLONG); + + uint64_t alloc_size = (size + 0xFFF) & ~0xFFFULL; + void *kbuf = (void*)talloc(alloc_size); + if (!kbuf) return SYSCALL_ERRNO(SYSCALL_ENOMEM); + + size_t r = simple_read(path, kbuf, size); + if (r) { + ur = copy_to_user(ctx, ubuf, kbuf, r); + if (ur != UACCESS_OK) { + temp_free(kbuf, alloc_size); + return ur; + } + } + + temp_free(kbuf, alloc_size); + return r; } u64 syscall_swritef(process_t *ctx){ - const char *path = (const char*)ctx->PROC_X0; - const void *buf = (void*)ctx->PROC_X1; + uintptr_t upath = (uintptr_t)ctx->PROC_X0; + uintptr_t ubuf = (uintptr_t)ctx->PROC_X1; size_t size = (size_t)ctx->PROC_X2; - return simple_write(path, buf, size); + + char path[255] = {}; + size_t copied = 0; + bool term = false; + uaccess_result_t ur = copy_str_from_user(ctx, path, sizeof(path), upath, &copied, &term); + if (ur != UACCESS_OK) return ur; + if (!term) return SYSCALL_ERRNO(SYSCALL_ENAMETOOLONG); + + uint64_t alloc_size = (size + 0xFFF) & ~0xFFFULL; + void *kbuf = (void*)talloc(alloc_size); + if (!kbuf) return SYSCALL_ERRNO(SYSCALL_ENOMEM); + + ur = copy_from_user(ctx, kbuf, ubuf, size); + if (ur != UACCESS_OK){ + temp_free(kbuf, alloc_size); + return ur; + } + + size_t r = simple_write(path, kbuf, size); + temp_free(kbuf, alloc_size); + return r; } u64 syscall_closef(process_t *ctx){ - file *descriptor = (file*)ctx->PROC_X0; - close_file(descriptor); + uintptr_t udesc = (uintptr_t)ctx->PROC_X0; + file descriptor = {}; + uaccess_result_t ur = copy_from_user(ctx, &descriptor, udesc, sizeof(descriptor)); + if (ur != UACCESS_OK) return ur; + close_file(&descriptor); return 0; } u64 syscall_dir_list(process_t *ctx){ - const char *path = (char *)ctx->PROC_X0; - void *buf = (void*)ctx->PROC_X1; + uintptr_t upath = (uintptr_t)ctx->PROC_X0; + uintptr_t ubuf = (uintptr_t)ctx->PROC_X1; size_t size = (size_t)ctx->PROC_X2; - u64 *offset = (u64*)ctx->PROC_X3; - return list_directory_contents(path, buf, size, offset); + uintptr_t uoffset = (uintptr_t)ctx->PROC_X3; + + char path[255] = {}; + size_t copied = 0; + bool term = false; + uaccess_result_t ur = copy_str_from_user(ctx, path, sizeof(path), upath, &copied, &term); + if (ur != UACCESS_OK) return ur; + if (!term) return SYSCALL_ERRNO(SYSCALL_ENAMETOOLONG); + + uint64_t off = 0; + ur = copy_from_user(ctx, &off, uoffset, sizeof(off)); + if (ur != UACCESS_OK) return ur; + + uint64_t alloc_size = (size + 0xFFF) & ~0xFFFULL; + void *kbuf = (void*)talloc(alloc_size); + if (!kbuf) return SYSCALL_ERRNO(SYSCALL_ENOMEM); + + size_t r = list_directory_contents(path, kbuf, size, &off); + if (r) { + ur = copy_to_user(ctx, ubuf, kbuf, r); + if (ur != UACCESS_OK) { + temp_free(kbuf, alloc_size); + return ur; + } + } + ur = copy_to_user(ctx, uoffset, &off, sizeof(off)); + temp_free(kbuf, alloc_size); + if (ur != UACCESS_OK) return ur; + return r; } // uint64_t syscall_load_fsmod(process_t *ctx){ @@ -293,6 +753,7 @@ Don't ever do that again\r\n\ syscall_entry syscalls[] = { [MALLOC_CODE] = syscall_malloc, [FREE_CODE] = syscall_free, + [BRK_CODE] = syscall_brk, [PALLOC_CODE] = syscall_palloc, [PFREE_CODE] = syscall_pfree, [PRINTL_CODE] = syscall_printl, @@ -351,21 +812,25 @@ void backtrace(uintptr_t fp, uintptr_t elr, sizedptr debug_line, sizedptr debug_ } for (uint8_t depth = 1; depth < 10 && fp; depth++) { - uintptr_t return_address = (*(uintptr_t*)(fp + 8)); - - if (return_address != 0){ - return_address -= 4;//Return address is the next instruction after branching - if (!decode_crash_address(depth, return_address, debug_line, debug_line_str)) - kprintf("%i: caller address: %llx", depth, return_address, return_address); - fp = *(uintptr_t*)fp; - if (!mmu_translate(fp)) return; - } else return; - + int tr_ra = 0; + uintptr_t ra_pa = mmu_translate(0, fp + 8, &tr_ra); + if (tr_ra) return; + + uintptr_t return_address = (*(uintptr_t*)dmap_pa_to_kva((paddr_t)ra_pa)); + if (!return_address) return; + return_address -= 4;//Return address is the next instruction after branching + if (!decode_crash_address(depth, return_address, debug_line, debug_line_str)) + kprintf("%i: caller address: %llx", depth, return_address); + int tr = 0; + uintptr_t fp_pa = mmu_translate(0, fp, &tr); + if (tr) return; + fp = *(uintptr_t*)dmap_pa_to_kva((paddr_t)fp_pa); } } const char* fault_messages[] = { [0b000000] = "Address size fault in TTBR0 or TTBR1", + [0b000100] = "Translation fault, 0th level", [0b000101] = "Translation fault, 1st level", [0b000110] = "Translation fault, 2nd level", [0b000111] = "Translation fault, 3rd level", @@ -390,7 +855,10 @@ const char* fault_messages[] = { void coredump(uintptr_t esr, uintptr_t elr, uintptr_t far, uintptr_t sp){ uint8_t ifsc = esr & 0x3F; - kprint(fault_messages[ifsc]); + const char *m = 0; + if (ifsc < 64) m = fault_messages[ifsc]; + if (!m) m = "Unknown fault"; + kprint(m); process_t *proc = get_current_proc(); backtrace(sp, elr, proc->debug_lines, proc->debug_line_str); @@ -399,11 +867,12 @@ void coredump(uintptr_t esr, uintptr_t elr, uintptr_t far, uintptr_t sp){ if (far > 0) debug_mmu_address(far); else - kprintf("Null pointer accessed at %x",elr); + kprintf("Null pointer accessed at %llx", elr); } void sync_el0_handler_c(){ save_return_address_interrupt(); + mmu_ttbr0_disable_user(); syscall_depth++; #if TEST @@ -428,20 +897,27 @@ void sync_el0_handler_c(){ uint64_t far; asm volatile ("mrs %0, far_el1" : "=r"(far)); + if (ec == 0x24 || ec == 0x20){ + if (mm_try_handle_page_fault(proc, far, esr)){ + syscall_depth--; + process_restore(); + } + } uint64_t result = 0; if (ec == 0x15) { - if (syscalls[iss]){ - result = syscalls[iss](proc); + syscall_entry entry = syscalls[iss]; + if (entry){ + result = entry(proc); } else { - kprintf("Unknown syscall in process. ESR: %x. ELR: %x. FAR: %x", esr, elr, far); + kprintf("Unknown syscall in process. ESR: %llx. ELR: %llx. FAR: %llx", esr, elr, far); coredump(esr, elr, far, proc->sp); syscall_depth--; stop_current_process(ec); } } else { if (far == 0 && elr == 0 && currentEL == 0){ - kprintf("Process has exited %x",x0); + kprintf("Process has exited %llx", x0); syscall_depth--; stop_current_process(x0); } else { @@ -449,7 +925,7 @@ void sync_el0_handler_c(){ case 0x20: case 0x21: { if (far == 0){ - kprintf("Process has exited %x",x0); + kprintf("Process has exited %llx", x0); syscall_depth--; stop_current_process(x0); } @@ -458,7 +934,11 @@ void sync_el0_handler_c(){ if (currentEL == 1){ if (syscall_depth < 3){ if (syscall_depth < 1) kprintf("System has crashed. ESR: %llx. ELR: %llx. FAR: %llx", esr, elr, far); - if (syscall_depth < 2) coredump(esr, elr, far, proc->sp); + if (syscall_depth < 2) { + uint64_t ksp = 0; + asm volatile ("mov %0, sp" : "=r"(ksp)); + coredump(esr, elr, far, ksp); + } handle_exception("UNEXPECTED EXCEPTION",ec); } while (true); diff --git a/kernel/process/uaccess.c b/kernel/process/uaccess.c new file mode 100644 index 00000000..9d22ea6f --- /dev/null +++ b/kernel/process/uaccess.c @@ -0,0 +1,129 @@ +#include "process/uaccess.h" +#include "memory/mm_process.h" +#include "memory/mmu.h" +#include "memory/addr.h" +#include "std/memory.h" + +bool access_ok_range(process_t *proc, uintptr_t addr, size_t size, bool want_write) { + if (!proc) return false; + if (!proc->mm.ttbr0) return false; + if (!size) return true; + + if ((addr >> 47) & 1) return false; + + uintptr_t end = addr + size - 1; + if (end < addr) return false; + if ((end >> 47) & 1) return false; + + uintptr_t cur = addr; + while (cur <= end) { + vma *m = mm_find_vma(&proc->mm, cur); + if (!m) return false; + if (want_write && !(m->prot & MEM_RW)) return false; + + uintptr_t next = m->end; + if (!next || next - 1 >= end) return true; + cur = next; + } + + return true; +} + +uaccess_result_t copy_from_user(process_t *proc, void *dst, uintptr_t src, size_t size) { + if (!dst && size) return UACCESS_EINVAL; + if (!size) return UACCESS_OK; + if (!access_ok_range(proc, src, size, false)) return UACCESS_EFAULT; + + uint8_t *d = (uint8_t*)dst; + + while (size) { + size_t off = src & (PAGE_SIZE - 1); + size_t chunk = PAGE_SIZE - off; + if (chunk > size) chunk = size; + + int st = 0; + uintptr_t pa =mmu_translate((uint64_t*)proc->mm.ttbr0, src, &st); + if (st) { + uint64_t esr = (0x24ULL << 26) | 0x7ULL; + if (!mm_try_handle_page_fault(proc, src, esr)) return UACCESS_EFAULT; + + pa = mmu_translate((uint64_t*)proc->mm.ttbr0, src, &st); + if (st) return UACCESS_EFAULT; + } + + memcpy(d, (const void*)dmap_pa_to_kva((paddr_t)pa), chunk); + d += chunk; + src += chunk; + size -= chunk; + } + + return UACCESS_OK; +} + +uaccess_result_t copy_to_user(process_t *proc, uintptr_t dst, const void *src, size_t size) { + if (!src && size) return UACCESS_EINVAL; + if (!size) return UACCESS_OK; + if (!access_ok_range(proc, dst, size, true)) return UACCESS_EFAULT; + + const uint8_t *s = (const uint8_t*)src; + + while (size) { + size_t off = dst & (PAGE_SIZE - 1); + size_t chunk = PAGE_SIZE - off; + if (chunk > size) chunk = size; + + int st = 0; + uintptr_t pa = mmu_translate((uint64_t*)proc->mm.ttbr0, dst, &st); + if (st) { + uint64_t esr = (0x24ULL << 26) | 0x7ULL | (1 << 6); + if (!mm_try_handle_page_fault(proc, dst, esr)) return UACCESS_EFAULT; + + pa = mmu_translate((uint64_t*)proc->mm.ttbr0, dst, &st); + if (st) return UACCESS_EFAULT; + } + + memcpy((void*)dmap_pa_to_kva((paddr_t)pa), s, chunk); + s += chunk; + dst += chunk; + size -= chunk; + } + + return UACCESS_OK; +} + +uaccess_result_t copy_str_from_user(process_t *proc, char *dst, size_t dst_size, uintptr_t src, size_t *out_copied, bool *out_terminated) { + if (out_copied) *out_copied = 0; + if (out_terminated) *out_terminated = false; + if (!dst || !dst_size) return UACCESS_EINVAL; + if (!proc || !proc->mm.ttbr0) return UACCESS_EFAULT; + if ((src >> 47) & 1) return UACCESS_EFAULT; + + size_t pos = 0; + + while (pos + 1 < dst_size) { + size_t chunk = PAGE_SIZE - ((src + pos) & (PAGE_SIZE - 1)); + if (chunk > dst_size - 1 - pos) chunk = dst_size - 1 - pos; + if (!access_ok_range(proc, src + pos, chunk, false)) return UACCESS_EFAULT; + + int st = 0; + uintptr_t pa = mmu_translate((uint64_t*)proc->mm.ttbr0, src + pos, &st); + if (st) { + if (!mm_try_handle_page_fault(proc, src + pos, (0x24ULL << 26) | 0x7ULL)) return UACCESS_EFAULT; + pa = mmu_translate((uint64_t*)proc->mm.ttbr0, src + pos, &st); + if (st) return UACCESS_EFAULT; + } + memcpy(dst + pos, (const void*)dmap_pa_to_kva((paddr_t)pa), chunk); + for (size_t i = 0; i < chunk; i++) { + if (dst[pos + i]) continue; + if (out_copied) *out_copied = pos + i +1; + if (out_terminated) *out_terminated = true; + return UACCESS_OK; + } + + pos += chunk; + } + + dst[dst_size-1] = 0; + if (out_copied) *out_copied = dst_size - 1; + return UACCESS_ENAMETOOLONG; +} \ No newline at end of file diff --git a/kernel/process/uaccess.h b/kernel/process/uaccess.h new file mode 100644 index 00000000..2bf1ce4f --- /dev/null +++ b/kernel/process/uaccess.h @@ -0,0 +1,19 @@ +#pragma once + +#include "types.h" +#include "syscalls/errno.h" +#include "process/process.h" + +typedef enum uaccess_result { + UACCESS_OK = 0, + UACCESS_EPERM = -SYSCALL_EPERM, + UACCESS_ENOMEM = -SYSCALL_ENOMEM, + UACCESS_EFAULT = -SYSCALL_EFAULT, + UACCESS_EINVAL = -SYSCALL_EINVAL, + UACCESS_ENAMETOOLONG = -SYSCALL_ENAMETOOLONG, +} uaccess_result_t; + +bool access_ok_range(process_t *proc, uintptr_t addr, size_t size, bool want_write); +uaccess_result_t copy_from_user(process_t *proc, void *dst, uintptr_t src, size_t size); +uaccess_result_t copy_to_user(process_t *proc, uintptr_t dst, const void *src, size_t size); +uaccess_result_t copy_str_from_user(process_t *proc, char *dst, size_t dst_size, uintptr_t src, size_t *out_copied, bool *out_terminated); diff --git a/kernel/sysregs.h b/kernel/sysregs.h index b93950e2..94f883d3 100644 --- a/kernel/sysregs.h +++ b/kernel/sysregs.h @@ -1,5 +1,4 @@ #pragma once - // *************************************** // SCTLR_EL1, System Control Register (EL1), Page 2654 of AArch64-Reference-Manual. // *************************************** @@ -38,7 +37,9 @@ #define SPSR_MASK_ALL (7 << 6) #define SPSR_EL1h (5 << 0) -#define SPSR3_VALUE (SPSR_MASK_ALL | SPSR_EL1h) +#define SPSR_EL2h (9 << 0) +#define SPSR3_VALUE (SPSR_MASK_ALL | SPSR_EL2h) +#define SCTLR_VALUE_EARLY (SCTLR_RESERVED | SCTLR_EE_LITTLE_ENDIAN | SCTLR_I_CACHE_DISABLED | SCTLR_D_CACHE_DISABLED | SCTLR_MMU_DISABLED) // *************************************** // CNTHCTL_EL2, Counter-timer Hypervisor Control Register (EL2) Page 9569 of AArch64-Reference-Manual. @@ -52,17 +53,26 @@ // MMU // *************************************** -//30 = Translation granule EL1. 10 = 4kb | 14 = TG EL0 00 = 4kb. 0xFFFFFFFF to translate to 64 -#define TCR_VALUE ((0xFFFFFFFF << 32) | ((64 - 48) << 0) | ((64 - 48) << 16) | (0b00 << 14) | (0b10 << 30)) +#define TCR_T0SZ (64-48) +#define TCR_T1SZ (64-48) +#define TCR_TG0_4KB (0b00) +#define TCR_TG1_4KB (0b10) +#define TCR_SH_INNER (0b11) +#define TCR_RGN_WBWA (0b01) + +#define TCR_VALUE_BASE (0ULL | (TCR_T0SZ << 0) | (TCR_T1SZ << 16) \ + | (TCR_TG0_4KB << 14) | (TCR_TG1_4KB << 30) | (TCR_SH_INNER << 12) \ + | (TCR_SH_INNER << 28) | (TCR_RGN_WBWA << 8) | (TCR_RGN_WBWA << 10) \ + | (TCR_RGN_WBWA << 24) | (TCR_RGN_WBWA << 26)) #define MAIR_DEVICE_nGnRnE 0b00000000 -#define MAIR_NORMAL_NOCACHE 0b01000100 +#define MAIR_NORMAL_WBWA 0xFF #define MAIR_IDX_DEVICE 0 #define MAIR_IDX_NORMAL 1 -#define MAIR_VALUE ((MAIR_DEVICE_nGnRnE << (MAIR_IDX_DEVICE * 8)) | (MAIR_NORMAL_NOCACHE << (MAIR_IDX_NORMAL * 8))) +#define MAIR_VALUE ((MAIR_DEVICE_nGnRnE << (MAIR_IDX_DEVICE * 8)) | (MAIR_NORMAL_WBWA << (MAIR_IDX_NORMAL * 8))) -#define HIGH_VA 0xFFFF000000000000ULL +#define HIGH_VA 0xFFFF800000000000ULL #define PHYS_TO_VIRT(x) (((uintptr_t)(x) != 0) ? ((uintptr_t)(x) | HIGH_VA) : 0) #define VIRT_TO_PHYS(x) (((uintptr_t)(x) != 0) ? ((uintptr_t)(x) & ~HIGH_VA) : 0) diff --git a/kernel/tests/allocation/alloc_tests.c b/kernel/tests/allocation/alloc_tests.c index 0c6dcbff..0d0629b9 100644 --- a/kernel/tests/allocation/alloc_tests.c +++ b/kernel/tests/allocation/alloc_tests.c @@ -6,16 +6,27 @@ bool test_kalloc_free(){ void *page = palloc(PAGE_SIZE, MEM_PRIV_KERNEL, MEM_RW, false); void *mem = kalloc(page, sizeof(uint64_t), ALIGN_16B, MEM_PRIV_KERNEL); kfree(mem, sizeof(uint64_t)); - assert_eq(*(uint64_t*)mem, sizeof(uint64_t), "Memory not freed: %x",*(uint64_t*)mem); + assert_eq(*(uint64_t*)mem, 0xDEADBEEFDEADBEEF, "Freed memory not poisoned: %llx", (uint64_t*)mem); + free_managed_page(page); return true; } bool test_kalloc_alignment_free(){ void *page = palloc(PAGE_SIZE, MEM_PRIV_KERNEL, MEM_RW, false); void *mem = kalloc(page, sizeof(uint8_t), ALIGN_16B, MEM_PRIV_KERNEL); - kfree(mem, sizeof(uint8_t)); - //potential fix: after aligning, put a free block on the upper part. Still, not fully freeing the memory before it - assert_eq(*(uint64_t*)mem, ALIGN_16B, "Aligned memory not fully freed: %x",*(uint64_t*)mem); + void *mem1 = kalloc(page, sizeof(uint8_t), ALIGN_16B, MEM_PRIV_KERNEL); + void *mem2 = kalloc(page, sizeof(uint8_t), ALIGN_16B, MEM_PRIV_KERNEL); + assert_eq((uintptr_t)mem & (ALIGN_16B - 1), 0, "a not aligned: %llx",(uint64_t)mem); + assert_eq((uintptr_t)mem1 & (ALIGN_16B - 1), 0, "mem1 not aligned: %llx", (uint64_t)mem1); + assert_eq((uintptr_t)mem2 & (ALIGN_16B - 1), 0, "mem2 not aligned: %llx", (uint64_t)mem2); + assert_true((uintptr_t)mem1 > (uintptr_t)mem + 1, "no alignment pad before b"); + assert_true((uintptr_t)mem2 > (uintptr_t)mem1 + 1, "no alignment pad after b"); + + kfree(mem1, sizeof(uint8_t)); + assert_eq(*(uint64_t*)mem1, 0xDEADBEEFDEADBEEFULL, "Aligned freed memory not poisoned: %llx", *(uint64_t*)mem1); + void *d = kalloc(page, 16, ALIGN_16B, MEM_PRIV_KERNEL); + assert_eq((uintptr_t)d, (uintptr_t)mem1, "Aligned span not reused: %llx", (uint64_t)d); + free_managed_page(page); return true; } @@ -24,6 +35,7 @@ bool test_page_kalloc_no_free_unmanaged(){ void *mem = kalloc(page, sizeof(uint64_t), ALIGN_4KB, MEM_PRIV_KERNEL); pfree(page, PAGE_SIZE); assert_true(page_used((uintptr_t)mem), "Page should not have been freed: %x",(uint64_t)mem); + free_managed_page(page); return true; } @@ -35,6 +47,58 @@ bool test_page_kalloc_free_managed(){ return true; } +//NOTE these reuse tests assume a specific reuse policy +//if that policy changes, these tests may no longer succeed even if the reallocation works +bool test_palloc_reuse_single() { + void *page = palloc(PAGE_SIZE, MEM_PRIV_KERNEL, MEM_RW, false); + assert_true(page != 0, "single page allocation failed"); + assert_true(page_used((uintptr_t)page), "single page not marked used"); + pfree(page, PAGE_SIZE); + assert_false(page_used((uintptr_t)page), "single page not freed"); + void *page2 = palloc(PAGE_SIZE, MEM_PRIV_KERNEL, MEM_RW, false); + assert_eq((uintptr_t)page2, (uintptr_t)page, "single page not reused: %llx", (uintptr_t)page2); + pfree(page2, PAGE_SIZE); + return true; +} + +bool test_palloc_reuse_gap() { + void *a = palloc(PAGE_SIZE, MEM_PRIV_KERNEL, MEM_RW, false); + void *b = palloc(PAGE_SIZE, MEM_PRIV_KERNEL, MEM_RW, false); + void *c = palloc(PAGE_SIZE, MEM_PRIV_KERNEL, MEM_RW, false); + assert_true(a && b && c, "gap allocation failed"); + pfree(b, PAGE_SIZE); + void *d = palloc(PAGE_SIZE, MEM_PRIV_KERNEL, MEM_RW, false); + assert_eq((uintptr_t)d, (uintptr_t)b, "gap page not reused: %llx", (uint64_t)d); + pfree(a, PAGE_SIZE); + pfree(c, PAGE_SIZE); + pfree(d, PAGE_SIZE); + return true; +} + +bool test_palloc_large_reuse() { + size_t amount = PAGE_SIZE * 80; + uint64_t *mem = (uint64_t*)palloc(amount, MEM_PRIV_KERNEL, MEM_RW, false); + for (size_t i = 0; i < amount / sizeof(uint64_t); i += PAGE_SIZE / sizeof(uint64_t)) mem[i] = 0xA5A5000000000000ULL | i; + pfree(mem, amount); + uint64_t *mem2 = (uint64_t*)palloc(amount, MEM_PRIV_KERNEL, MEM_RW, false); + assert_eq((uintptr_t)mem2, (uintptr_t)mem, "allocation not reused: %llx", (uint64_t)mem2); + pfree(mem2, amount); + return true; +} + +bool test_kalloc_fragment_reuse() { + void *page = palloc(PAGE_SIZE, MEM_PRIV_KERNEL, MEM_RW, false); + void *a = kalloc(page, 64, ALIGN_16B, MEM_PRIV_KERNEL); + void *b = kalloc(page, 128, ALIGN_16B, MEM_PRIV_KERNEL); + void *c = kalloc(page, 64, ALIGN_16B, MEM_PRIV_KERNEL); + kfree(b, 128); + void *d = kalloc(page, 96, ALIGN_16B, MEM_PRIV_KERNEL); + assert_true((uintptr_t)d >= (uintptr_t)b && (uintptr_t)d + 96 <= (uintptr_t)b + 128, "allocation not placed inside freed fragment: %llx", (uint64_t)d); + + free_managed_page(page); + return true; +} + bool test_after_free(){ uint64_t *a = malloc(64); a[3] = 12345678; @@ -46,6 +110,10 @@ bool test_after_free(){ bool alloc_tests(){ return test_after_free() && + test_palloc_reuse_single() && + test_palloc_reuse_gap() && + test_palloc_large_reuse() && + test_kalloc_fragment_reuse() && test_kalloc_free() && test_page_kalloc_free_managed() && test_page_kalloc_no_free_unmanaged() && diff --git a/kernel/virtio/virtio_pci.c b/kernel/virtio/virtio_pci.c index ca98150a..76c1259d 100644 --- a/kernel/virtio/virtio_pci.c +++ b/kernel/virtio/virtio_pci.c @@ -1,5 +1,6 @@ #include "console/kio.h" #include "pci.h" +#include "std/memory.h" #include "std/memory_access.h" #include "memory/page_allocator.h" #include "virtio_pci.h" @@ -8,12 +9,6 @@ //TODO implement proper virtqueue handling w/ descriptor allocation, reuse and support for multiple in-flight requests using used.ring completions -#define VIRTIO_STATUS_RESET 0x0 -#define VIRTIO_STATUS_ACKNOWLEDGE 0x1 -#define VIRTIO_STATUS_DRIVER 0x2 -#define VIRTIO_STATUS_DRIVER_OK 0x4 -#define VIRTIO_STATUS_FEATURES_OK 0x8 -#define VIRTIO_STATUS_FAILED 0x80 #define VIRTIO_PCI_CAP_COMMON_CFG 1 #define VIRTIO_PCI_CAP_NOTIFY_CFG 2 @@ -22,8 +17,6 @@ #define VIRTIO_PCI_CAP_PCI_CFG 5 #define VIRTIO_PCI_CAP_VENDOR_CFG 9 -#define VIRTQ_DESC_F_NEXT 1 - struct virtio_pci_cap { uint8_t cap_vndr; uint8_t cap_next; @@ -36,12 +29,8 @@ struct virtio_pci_cap { uint32_t length; }; -#define VIRTQ_DESC_F_NEXT 1 -#define VIRTQ_DESC_F_WRITE 2 - -static bool virtio_verbose = false; - -static uint64_t feature_mask = 0; +bool virtio_verbose = false; +uint64_t feature_mask = 0; void virtio_enable_verbose(){ virtio_verbose = true; @@ -59,6 +48,17 @@ void virtio_set_feature_mask(uint64_t mask){ } void virtio_get_capabilities(virtio_device *dev, uint64_t pci_addr, uint64_t *mmio_start, uint64_t *mmio_size) { + if (!dev) return; + + dev->common_cfg = 0; + dev->notify_cfg = 0; + dev->device_cfg = 0; + dev->isr_cfg = 0; + dev->notify_off_multiplier = 0; + + if (mmio_start) *mmio_start = 0; + if (mmio_size) *mmio_size = 0; + uint64_t offset = read32(pci_addr + 0x34); while (offset) { uint64_t cap_addr = pci_addr + offset; @@ -81,17 +81,17 @@ void virtio_get_capabilities(virtio_device *dev, uint64_t pci_addr, uint64_t *mm } if (cap->cfg_type == VIRTIO_PCI_CAP_COMMON_CFG){ - dev->common_cfg = (struct virtio_pci_common_cfg*)PHYS_TO_VIRT((bar_base + cap->offset)); + dev->common_cfg = (volatile struct virtio_pci_common_cfg*)PHYS_TO_VIRT((bar_base + cap->offset)); kprintfv("[VIRTIO] Common CFG @ %llx",dev->common_cfg); } else if (cap->cfg_type == VIRTIO_PCI_CAP_NOTIFY_CFG) { - dev->notify_cfg = (uint8_t*)PHYS_TO_VIRT((bar_base + cap->offset)); + dev->notify_cfg = (volatile uint8_t*)PHYS_TO_VIRT((bar_base + cap->offset)); kprintfv("[VIRTIO] Notify CFG @ %llx",dev->notify_cfg); - dev->notify_off_multiplier = *(uint32_t*)PHYS_TO_VIRT((cap_addr + sizeof(struct virtio_pci_cap))); + dev->notify_off_multiplier = *(volatile uint32_t*)PHYS_TO_VIRT((cap_addr + sizeof(struct virtio_pci_cap))); } else if (cap->cfg_type == VIRTIO_PCI_CAP_DEVICE_CFG){ - dev->device_cfg = (uint8_t*)PHYS_TO_VIRT((bar_base + cap->offset)); + dev->device_cfg = (volatile uint8_t*)PHYS_TO_VIRT((bar_base + cap->offset)); kprintfv("[VIRTIO] Device CFG @ %llx",dev->device_cfg); } else if (cap->cfg_type == VIRTIO_PCI_CAP_ISR_CFG){ - dev->isr_cfg = (uint8_t*)PHYS_TO_VIRT((bar_base + cap->offset)); + dev->isr_cfg = (volatile uint8_t*)PHYS_TO_VIRT((bar_base + cap->offset)); kprintfv("[VIRTIO] ISR CFG @ %llx",dev->isr_cfg); } } @@ -101,13 +101,22 @@ void virtio_get_capabilities(virtio_device *dev, uint64_t pci_addr, uint64_t *mm } bool virtio_init_device(virtio_device *dev) { + if (!dev || !dev->common_cfg) return false; + + volatile struct virtio_pci_common_cfg* cfg = dev->common_cfg; + + memset(dev->queues, 0, sizeof(dev->queues)); + dev->num_queues = 0; + dev->current_queue = 0; + dev->negotiated_features = 0; + uint32_t timeout = 2000; + while (cfg->device_status != 0) { + if (timeout == 0) return false; + timeout--; + delay(1); + } - struct virtio_pci_common_cfg* cfg = dev->common_cfg; - - cfg->device_status = 0; - if (!wait((uint32_t*)&cfg->device_status, 0, false, 2000)) return false; - - cfg->device_status |= VIRTIO_STATUS_ACKNOWLEDGE; + cfg->device_status = VIRTIO_STATUS_ACKNOWLEDGE; cfg->device_status |= VIRTIO_STATUS_DRIVER; cfg->device_feature_select = 0; @@ -142,9 +151,18 @@ bool virtio_init_device(virtio_device *dev) { if (!dev->status_dma) return false; *dev->status_dma = 0; - uint32_t queue_index = 0; - uint32_t size; - while ((size = select_queue(dev,queue_index))){ + dev->num_queues = cfg->num_queues; + if (dev->num_queues > VIRTIO_MAX_QUEUES) dev->num_queues = VIRTIO_MAX_QUEUES; + + for (uint16_t queue_index = 0; queue_index < dev->num_queues; queue_index++) { + cfg->queue_select = queue_index; + asm volatile ("dsb sy" ::: "memory"); + uint16_t size = cfg->queue_size; + dev->queues[queue_index].size = size; + dev->queues[queue_index].notify_off = cfg->queue_notify_off; + dev->queues[queue_index].notify_data = cfg->queue_notify_data; + + if (!size) continue; uint64_t desc_sz = 16ULL * size; uint64_t avail_sz = 6ULL + 2ULL * size; uint64_t used_sz = 6ULL + 8ULL * size; @@ -158,46 +176,82 @@ bool virtio_init_device(virtio_device *dev) { void* used = palloc(used_alloc, MEM_PRIV_KERNEL, MEM_DEV | MEM_RW, true); if (!base || !avail || !used) return false; - dev->common_cfg->queue_desc = VIRT_TO_PHYS((uint64_t)base); - dev->common_cfg->queue_driver = VIRT_TO_PHYS((uint64_t)avail); - dev->common_cfg->queue_device = VIRT_TO_PHYS((uint64_t)used); - - volatile virtq_avail* A = (volatile virtq_avail*)(uintptr_t)avail; - A->flags = 0; - A->idx = 0; - - volatile virtq_used* U = (volatile virtq_used*)(uintptr_t)used; - U->flags = 0; - U->idx = 0; - - dev->common_cfg->queue_enable = 1; - queue_index++; + memset(base, 0, desc_alloc); + memset(avail, 0, avail_alloc); + memset(used, 0, used_alloc); + uint64_t desc_pa = VIRT_TO_PHYS((uint64_t)base); + uint64_t driver_pa = VIRT_TO_PHYS((uint64_t)avail); + uint64_t device_pa = VIRT_TO_PHYS((uint64_t)used); + + cfg->queue_size = size; + cfg->queue_desc_lo = (uint32_t)desc_pa; + cfg->queue_desc_hi = (uint32_t)(desc_pa >> 32); + cfg->queue_driver_lo = (uint32_t)driver_pa; + cfg->queue_driver_hi = (uint32_t)(driver_pa >> 32); + cfg->queue_device_lo = (uint32_t)device_pa; + cfg->queue_device_hi = (uint32_t)(device_pa >> 32); + cfg->queue_enable = 1; + + dev->queues[queue_index].valid = true; + dev->queues[queue_index].desc_pa = desc_pa; + dev->queues[queue_index].driver_pa = driver_pa; + dev->queues[queue_index].device_pa = device_pa; + dev->queues[queue_index].desc = (volatile virtq_desc*)base; + dev->queues[queue_index].driver = (volatile virtq_avail*)avail; + dev->queues[queue_index].device = (volatile virtq_used*)used; } - kprintfv("Device initialized %i virtqueues",queue_index); + kprintfv("Device initialized %i virtqueues", dev->num_queues); - select_queue(dev,0); + for (uint16_t queue_index = 0; queue_index < dev->num_queues; queue_index++) { + if (!dev->queues[queue_index].valid) continue; + dev->current_queue = queue_index; + cfg->queue_select = queue_index; + asm volatile ("dsb sy" ::: "memory"); + return true; + } - cfg->device_status |= VIRTIO_STATUS_DRIVER_OK; - return true; + return false; } uint32_t select_queue(virtio_device *dev, uint32_t index){ - dev->common_cfg->queue_select = index; + if (!dev || !dev->common_cfg) return 0; + dev->current_queue = (uint16_t)index; + dev->common_cfg->queue_select = (uint16_t)index; asm volatile ("dsb sy" ::: "memory"); - return dev->common_cfg->queue_size; + if (index >= VIRTIO_MAX_QUEUES) return dev->common_cfg->queue_size; + return dev->queues[index].size; +} + +void virtio_notify(virtio_device *dev) { + if (!dev || !dev->notify_cfg) return; + uint16_t index = dev->current_queue; + if (index >= VIRTIO_MAX_QUEUES) return; + if (!dev->queues[index].valid) return; + + uint32_t mul = dev->notify_off_multiplier; + if (!mul) mul = 1; + + uint16_t off = dev->queues[index].notify_off; + uint16_t value = (dev->negotiated_features & (1ULL << VIRTIO_F_NOTIFICATION_DATA)) ? dev->queues[index].notify_data : index; + + *(volatile uint16_t*)((uintptr_t)dev->notify_cfg + (uint64_t)off * (uint64_t)mul) = value; } bool virtio_send_nd(virtio_device *dev, const virtio_buf *bufs, uint16_t n) { if (!dev || !bufs || !n) return false; - uint16_t qsz = dev->common_cfg->queue_size; - if (!qsz || n > qsz) return false; + if (dev->current_queue >= VIRTIO_MAX_QUEUES) return false; + virtio_queue *queue = &dev->queues[dev->current_queue]; + if (!queue->valid || !queue->size || n > queue->size) return false; - volatile virtq_desc* d = PHYS_TO_VIRT_P((virtq_desc*)dev->common_cfg->queue_desc); - volatile virtq_avail* a = PHYS_TO_VIRT_P((virtq_avail*)dev->common_cfg->queue_driver); - volatile virtq_used* u = PHYS_TO_VIRT_P((virtq_used*)dev->common_cfg->queue_device); + volatile virtq_desc* d = queue->desc; + volatile virtq_avail* a = queue->driver; + volatile virtq_used* u = queue->device; + if (!d || !a || !u) return false; + + uint16_t qsz = queue->size; uint16_t last_used_idx = u->idx; for (uint16_t i = 0; i < n; ++i) { @@ -227,9 +281,15 @@ bool virtio_send_nd(virtio_device *dev, const virtio_buf *bufs, uint16_t n) { } void virtio_add_buffer(virtio_device *dev, uint16_t index, uint64_t buf, uint32_t buf_len, bool host_to_dev) { + if (!dev) return; + if (dev->current_queue >= VIRTIO_MAX_QUEUES) return; + + virtio_queue *queue = &dev->queues[dev->current_queue]; + if (!queue->valid || !queue->size) return; - volatile virtq_desc* d = PHYS_TO_VIRT_P((virtq_desc*)dev->common_cfg->queue_desc); - volatile virtq_avail* a = PHYS_TO_VIRT_P((virtq_avail*)dev->common_cfg->queue_driver); + volatile virtq_desc* d = queue->desc; + volatile virtq_avail* a = queue->driver; + if (!d || !a) return; d[index].addr = VIRT_TO_PHYS(buf); d[index].len = buf_len; @@ -237,7 +297,7 @@ void virtio_add_buffer(virtio_device *dev, uint16_t index, uint64_t buf, uint32_ d[index].next = 0; asm volatile ("dmb ishst" ::: "memory"); - a->ring[a->idx % dev->common_cfg->queue_size] = index; + a->ring[a->idx % queue->size] = index; asm volatile ("dmb ishst" ::: "memory"); a->idx++; asm volatile ("dmb ishst" ::: "memory"); diff --git a/kernel/virtio/virtio_pci.h b/kernel/virtio/virtio_pci.h index 2f5266b4..2744b651 100644 --- a/kernel/virtio/virtio_pci.h +++ b/kernel/virtio/virtio_pci.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "types.h" @@ -14,6 +14,15 @@ extern "C" { #define VIRTIO_F_VERSION_1 32 #define VIRTIO_F_NOTIFICATION_DATA 38 +#define VIRTIO_STATUS_RESET 0x0 +#define VIRTIO_STATUS_ACKNOWLEDGE 0x1 +#define VIRTIO_STATUS_DRIVER 0x2 +#define VIRTIO_STATUS_DRIVER_OK 0x4 +#define VIRTIO_STATUS_FEATURES_OK 0x8 +#define VIRTIO_STATUS_FAILED 0x80 + +#define VIRTIO_MAX_QUEUES 16 + typedef struct virtio_pci_common_cfg { uint32_t device_feature_select; uint32_t device_feature; @@ -28,9 +37,12 @@ typedef struct virtio_pci_common_cfg { uint16_t queue_msix_vector; uint16_t queue_enable; uint16_t queue_notify_off; - uint64_t queue_desc; - uint64_t queue_driver; - uint64_t queue_device; + uint32_t queue_desc_lo; + uint32_t queue_desc_hi; + uint32_t queue_driver_lo; + uint32_t queue_driver_hi; + uint32_t queue_device_lo; + uint32_t queue_device_hi; uint16_t queue_notify_data; uint16_t queue_reset; }__attribute__((packed)) virtio_pci_common_cfg; @@ -59,15 +71,31 @@ typedef struct { virtq_used_elem ring[]; }__attribute__((packed)) virtq_used; +typedef struct virtio_queue { + bool valid; + uint16_t size; + uint16_t notify_off; + uint16_t notify_data; + uint64_t desc_pa; + uint64_t driver_pa; + uint64_t device_pa; + volatile virtq_desc *desc; + volatile virtq_avail *driver; + volatile virtq_used *device; +} virtio_queue; + typedef struct virtio_device { - struct virtio_pci_common_cfg* common_cfg; - uint8_t* notify_cfg; - uint8_t* device_cfg; - uint8_t* isr_cfg; + volatile struct virtio_pci_common_cfg* common_cfg; + volatile uint8_t* notify_cfg; + volatile uint8_t* device_cfg; + volatile uint8_t* isr_cfg; uint32_t notify_off_multiplier; void *memory_page; uint8_t* status_dma; uint64_t negotiated_features; + uint16_t num_queues; + uint16_t current_queue; + virtio_queue queues[VIRTIO_MAX_QUEUES]; } virtio_device; typedef struct { @@ -78,19 +106,7 @@ typedef struct { #define VBUF(a,l,f) ((virtio_buf){.addr = (uint64_t)(a), .len = (uint32_t)(l), .flags = (uint16_t)(f)}) -static inline void virtio_notify(virtio_device *dev){ - if(!dev) return; - if(!dev->common_cfg) return; - if(!dev->notify_cfg) return; - - uint16_t off=dev->common_cfg->queue_notify_off; - uint32_t mul=dev->notify_off_multiplier; - if(!mul) mul=1; - - uint16_t v=(dev->negotiated_features&(1ULL<common_cfg->queue_notify_data:dev->common_cfg->queue_select; - *(volatile uint16_t*)(dev->notify_cfg+(uint64_t)off*(uint64_t)mul)=v; -} - +void virtio_notify(virtio_device *dev); void virtio_set_feature_mask(uint64_t mask); void virtio_enable_verbose(); void virtio_get_capabilities(virtio_device *dev, uint64_t pci_addr, uint64_t *mmio_start, uint64_t *mmio_size); diff --git a/modules/audio/virt/audio.cpp b/modules/audio/virt/audio.cpp index b5831673..65353ec4 100644 --- a/modules/audio/virt/audio.cpp +++ b/modules/audio/virt/audio.cpp @@ -3,6 +3,7 @@ #include "kernel_processes/kprocess_loader.h" #include "syscalls/syscalls.h" #include "exceptions/timer.h" +#include "exceptions/exception_handler.h" #include "math/math.h" #include "audio/cuatro.h" #include "audio/mixer.h" @@ -13,18 +14,30 @@ VirtioAudioDriver *audio_driver; bool init_audio(){ audio_driver = new VirtioAudioDriver(); - return audio_driver->init(); + if (!audio_driver) return false; + if (!audio_driver->init()) { + audio_driver = nullptr; + return false; + } + if (!audio_driver->out_dev) { + audio_driver = nullptr; + return false; + } + return true; } sizedptr audio_request_buffer(uint32_t device){ + if (!audio_driver || !audio_driver->out_dev) panic("audio not ready", 0); return audio_driver->out_dev->request_buffer(); } void audio_submit_buffer(){ + if (!audio_driver || !audio_driver->out_dev) panic("audio not ready", 0); audio_driver->out_dev->submit_buffer(audio_driver); } void audio_get_info(uint32_t* rate, uint8_t* channels) { + if (!audio_driver || !audio_driver->out_dev) panic("audio not ready", 0); *rate = audio_driver->out_dev->rate; *channels = audio_driver->out_dev->channels; } @@ -158,6 +171,7 @@ static int audio_mixer(int argc, char* argv[]){ } process_t* init_audio_mixer(){ + if (!audio_driver || !audio_driver->out_dev) return 0; return create_kernel_process("Audio out", audio_mixer, 0, 0); } diff --git a/modules/audio/virt/virtio_audio_pci.cpp b/modules/audio/virt/virtio_audio_pci.cpp index e7864667..4b8ad387 100644 --- a/modules/audio/virt/virtio_audio_pci.cpp +++ b/modules/audio/virt/virtio_audio_pci.cpp @@ -2,6 +2,7 @@ #include "pci.h" #include "console/kio.h" #include "memory/page_allocator.h" +#include "memory/addr.h" #include "std/memory_access.h" #include "syscalls/syscalls.h" #include "audio/audio.h" @@ -118,7 +119,9 @@ bool VirtioAudioDriver::init(){ select_queue(&audio_dev, CONTROL_QUEUE); - return get_config(); + bool ok = get_config(); + if (ok) audio_dev.common_cfg->device_status |= VIRTIO_STATUS_DRIVER_OK; + return ok; } bool VirtioAudioDriver::get_config(){ @@ -167,50 +170,60 @@ bool VirtioAudioDriver::config_streams(uint32_t streams){ uint8_t *streams_bytes = (uint8_t*)(resp + sizeof(virtio_snd_hdr)); virtio_snd_pcm_info *stream_info = (virtio_snd_pcm_info*)streams_bytes; + bool found_output = false; for (uint32_t stream = 0; stream < streams; stream++){ uint64_t format = read_unaligned64(&stream_info[stream].formats); uint64_t rate = read_unaligned64(&stream_info[stream].rates); - kprintf("[VIRTIO_AUDIO] Stream %i (%s): Features %x. Format %x. Sample %x. Channels %i-%i",stream, (uintptr_t)(stream_info[stream].direction ? "IN" : "OUT"), stream_info[stream].features, format, rate, stream_info->channels_min, stream_info->channels_max); + kprintf("[VIRTIO_AUDIO] Stream %i (%s): Features %x. Format %x. Sample %x. Channels %i-%i",stream, (uintptr_t)(stream_info[stream].direction ? "IN" : "OUT"), stream_info[stream].features, format, rate, stream_info[stream].channels_min, stream_info[stream].channels_max); - if (!(format & (1 << SND_SAMPLE_FORMAT))){ - kprintf("[VIRTIO_AUDIO implementation error] stream does not support int16 format"); - return false; - } + if (stream_info[stream].direction != VIRTIO_SND_D_OUTPUT) continue; + if (!(format & (1 << SND_SAMPLE_FORMAT))) continue; uint32_t sample_rate = 44100; if (!(rate & (1 << SND_SAMPLE_RATE))){ kprintf("[VIRTIO_AUDIO implementation error] stream does not support 44.1 kHz sample rate"); - return false; + continue; } - uint8_t channels = stream_info->channels_max; + uint8_t channels = stream_info[stream].channels_max; if (!stream_set_params(stream, stream_info[stream].features, SND_SAMPLE_FORMAT, SND_SAMPLE_RATE, channels)){ kprintf("[VIRTIO_AUDIO error] Failed to configure stream %i",stream); + continue; } - if (stream_info[stream].direction == VIRTIO_SND_D_OUTPUT){ - out_dev = new OutputAudioDevice(); - out_dev->stream_id = stream; - out_dev->rate = sample_rate; - out_dev->channels = channels; - out_dev->packet_size = sizeof(virtio_snd_pcm_xfer) + TOTAL_BUF_SIZE; - out_dev->buf_size = TOTAL_BUF_SIZE/SND_SAMPLE_BYTES; - out_dev->header_size = sizeof(virtio_snd_pcm_xfer); - out_dev->populate(); - } + out_dev = new OutputAudioDevice(); + out_dev->stream_id = stream; + out_dev->rate = sample_rate; + out_dev->channels = channels; + out_dev->packet_size = sizeof(virtio_snd_pcm_xfer) + TOTAL_BUF_SIZE; + out_dev->buf_size = TOTAL_BUF_SIZE/SND_SAMPLE_BYTES; + out_dev->header_size = sizeof(virtio_snd_pcm_xfer); + out_dev->populate(); + found_output = true; + break; } + + kfree(cmd, sizeof(virtio_snd_query_info)); + kfree((void*)resp, resp_size); + if (!found_output) return false; + select_queue(&audio_dev, TRANSMIT_QUEUE); return true; } void VirtioAudioDriver::send_buffer(sizedptr buf){ - virtio_add_buffer(&audio_dev, cmd_index % audio_dev.common_cfg->queue_size, buf.ptr, buf.size, true); - volatile virtq_used* u = (virtq_used*)audio_dev.common_cfg->queue_device; - while (u->idx < cmd_index-2) - yield(); + if (TRANSMIT_QUEUE >= audio_dev.num_queues) return; + if (!audio_dev.queues[TRANSMIT_QUEUE].valid || !audio_dev.queues[TRANSMIT_QUEUE].size) return; + + select_queue(&audio_dev, TRANSMIT_QUEUE); + uint16_t qsz = audio_dev.queues[TRANSMIT_QUEUE].size; + uint16_t index = cmd_index % qsz; + virtio_add_buffer(&audio_dev, index, buf.ptr, buf.size, true); cmd_index++; + volatile virtq_used* u = audio_dev.queues[TRANSMIT_QUEUE].device; + while (u && (uint16_t)(cmd_index - u->idx) > 2) yield(); } typedef struct virtio_snd_pcm_set_params { diff --git a/modules/audio/virt/virtio_audio_pci.hpp b/modules/audio/virt/virtio_audio_pci.hpp index 969faec0..5f14cb65 100644 --- a/modules/audio/virt/virtio_audio_pci.hpp +++ b/modules/audio/virt/virtio_audio_pci.hpp @@ -17,7 +17,7 @@ class VirtioAudioDriver: public AudioDriver { bool init(); void send_buffer(sizedptr buf) override; - AudioDevice *out_dev;//TODO: proper device management + AudioDevice *out_dev = nullptr;//TODO: proper device management private: bool get_config(); void config_jacks(); @@ -32,5 +32,5 @@ class VirtioAudioDriver: public AudioDriver { uint16_t cmd_index = 0; - virtio_device audio_dev; + virtio_device audio_dev = {}; }; \ No newline at end of file diff --git a/modules/disk/raspi/sdhci.cpp b/modules/disk/raspi/sdhci.cpp index a76b961b..838af754 100644 --- a/modules/disk/raspi/sdhci.cpp +++ b/modules/disk/raspi/sdhci.cpp @@ -134,7 +134,7 @@ bool SDHCI::setup_clock(){ bool SDHCI::init() { if (!SDHCI_BASE) return false; - register_device_memory(SDHCI_BASE, SDHCI_BASE); + register_device_memory_dmap(SDHCI_BASE); regs = (sdhci_regs*)SDHCI_BASE; diff --git a/modules/disk/virt/disk.c b/modules/disk/virt/disk.c index cfbeaad2..842984f1 100644 --- a/modules/disk/virt/disk.c +++ b/modules/disk/virt/disk.c @@ -55,6 +55,7 @@ bool init_disk_device(){ return false; } + blk_dev.common_cfg->device_status |= VIRTIO_STATUS_DRIVER_OK; return true; } diff --git a/modules/graph/raspi/videocore.cpp b/modules/graph/raspi/videocore.cpp index eacf88a7..27b35e35 100644 --- a/modules/graph/raspi/videocore.cpp +++ b/modules/graph/raspi/videocore.cpp @@ -11,6 +11,7 @@ #include "memory/mmu.h" #include "memory/page_allocator.h" #include "mailbox/vc_mbox.h" +#include "sysregs.h" static constexpr uint32_t RGB_FORMAT_XRGB8888 = ((uint32_t)('X') | ((uint32_t)('R') << 8) | ((uint32_t)('2') << 16) | ((uint32_t)('4') << 24)); @@ -54,7 +55,8 @@ bool VideoCoreGPUDriver::init(gpu_size preferred_screen_size){ kprintf("[VIDEOCORE] Failed updating mailbox"); return false; } - framebuffer = (uint32_t*)(uintptr_t)BUS_ADDRESS(fb_bus); + uint32_t fb_phys = BUS_ADDRESS(fb_bus); + framebuffer = (uint32_t*)(uintptr_t)PHYS_TO_VIRT(fb_phys); framebuffer_size = fb_size/2; if (fb_size < virt_h * virt_w * bpp){ kprintf("[VIDEOCORE] Fallback to one framebuffer with copying. Expected %i, got %i",virt_h * virt_w * bpp,fb_size); @@ -64,8 +66,8 @@ bool VideoCoreGPUDriver::init(gpu_size preferred_screen_size){ kprintf("[VIDEOCORE] Size %ix%i (%ix%i) (%ix%i) | %i (%i)",phys_w,phys_h,virt_w,virt_h,screen_size.width,screen_size.height,depth, stride); kprintf("[VIDEOCORE] Framebuffer allocated to %x (%i). BPP %i. Stride %i. Backbuffer at %x",framebuffer, framebuffer_size, bpp, stride/bpp,back_framebuffer); - mark_used((uintptr_t)framebuffer,count_pages(fb_size,PAGE_SIZE)); - for (size_t i = (uintptr_t)framebuffer; i < (uintptr_t)framebuffer + fb_size; i += GRANULE_4KB){ + mark_used((uintptr_t)fb_phys,count_pages(fb_size,PAGE_SIZE)); + for (size_t i = (size_t)fb_phys; i < (size_t)fb_phys + fb_size; i += GRANULE_4KB){ register_device_memory(i,i); } diff --git a/modules/graph/virt/virtio_gpu_pci.cpp b/modules/graph/virt/virtio_gpu_pci.cpp index 0b154bb7..302f85f0 100644 --- a/modules/graph/virt/virtio_gpu_pci.cpp +++ b/modules/graph/virt/virtio_gpu_pci.cpp @@ -73,7 +73,7 @@ bool VirtioGPUDriver::init(gpu_size preferred_screen_size){ resource_id_counter = 0; framebuffer_size = screen_size.width * screen_size.height * BPP; - framebuffer = VIRT_TO_PHYS((uintptr_t)kalloc(gpu_dev.memory_page, framebuffer_size, ALIGN_4KB, MEM_PRIV_KERNEL)); + framebuffer = (uintptr_t)kalloc(gpu_dev.memory_page, framebuffer_size, ALIGN_4KB, MEM_PRIV_KERNEL); ctx = { .dirty_rects = {}, @@ -96,7 +96,7 @@ bool VirtioGPUDriver::init(gpu_size preferred_screen_size){ return false; } - if (!attach_backing(fb_resource_id, (sizedptr){framebuffer,framebuffer_size})){ + if (!attach_backing(fb_resource_id, (sizedptr){VIRT_TO_PHYS(framebuffer),framebuffer_size})){ kprintf("[VIRTIO_GPU error] failed to attach backing"); return false; } @@ -106,6 +106,7 @@ bool VirtioGPUDriver::init(gpu_size preferred_screen_size){ else kprintf("[VIRTIO_GPU error] GPU did not return valid scanout data"); + gpu_dev.common_cfg->device_status |= VIRTIO_STATUS_DRIVER_OK; return true; } diff --git a/modules/graph/virt/virtio_gpu_pci.hpp b/modules/graph/virt/virtio_gpu_pci.hpp index 6f9bcf88..831fb190 100644 --- a/modules/graph/virt/virtio_gpu_pci.hpp +++ b/modules/graph/virt/virtio_gpu_pci.hpp @@ -56,7 +56,7 @@ typedef struct virtio_transfer_cmd { class VirtioGPUDriver : public GPUDriver { public: static VirtioGPUDriver* try_init(gpu_size preferred_screen_size); - VirtioGPUDriver(){} + VirtioGPUDriver() = default; bool init(gpu_size preferred_screen_size) override; void flush() override; @@ -83,9 +83,9 @@ class VirtioGPUDriver : public GPUDriver { private: gpu_size screen_size; - virtio_device gpu_dev; - uintptr_t framebuffer; - uint64_t framebuffer_size; + virtio_device gpu_dev = {}; + uintptr_t framebuffer = 0; + uint64_t framebuffer_size = 0; gpu_size get_display_info(); bool create_2d_resource(uint32_t resource_id, gpu_size size); @@ -96,20 +96,22 @@ class VirtioGPUDriver : public GPUDriver { uint32_t new_resource_id(); uint32_t new_cursor(uint32_t color); - uint32_t resource_id_counter; + uint32_t resource_id_counter = 0; - uint32_t fb_resource_id; - uint32_t cursor_resource_id; - uint32_t cursor_pressed_resource_id; - uint32_t cursor_unpressed_resource_id; + uint32_t fb_resource_id = 0; + uint32_t cursor_resource_id = 0; + uint32_t cursor_pressed_resource_id = 0; + uint32_t cursor_unpressed_resource_id = 0; - virtio_gpu_ctrl_hdr *trans_resp, *flush_resp, *cursor_resp; - virtio_gpu_update_cursor *cursor_cmd; - virtio_transfer_cmd *trans_cmd; - virtio_flush_cmd *flush_cmd; + virtio_gpu_ctrl_hdr *trans_resp = nullptr; + virtio_gpu_ctrl_hdr *flush_resp = nullptr; + virtio_gpu_ctrl_hdr *cursor_resp = nullptr; + virtio_gpu_update_cursor *cursor_cmd = nullptr; + virtio_transfer_cmd *trans_cmd = nullptr; + virtio_flush_cmd *flush_cmd = nullptr; - draw_ctx ctx; + draw_ctx ctx = {}; - bool scanout_found; - uint64_t scanout_id; + bool scanout_found = false; + uint64_t scanout_id = 0; }; \ No newline at end of file diff --git a/modules/mailbox/raspi/mailbox.c b/modules/mailbox/raspi/mailbox.c index 9a17c073..ebcebd04 100644 --- a/modules/mailbox/raspi/mailbox.c +++ b/modules/mailbox/raspi/mailbox.c @@ -4,7 +4,7 @@ #include "async.h" void mailbox_init(){ - register_device_memory(MAILBOX_BASE, MAILBOX_BASE); + register_device_memory_dmap(MAILBOX_BASE); } int mailbox_call(volatile uint32_t* mbox, uint8_t channel) { diff --git a/modules/serial/common/uart_base.c b/modules/serial/common/uart_base.c index c1bb83f0..4fe0af73 100644 --- a/modules/serial/common/uart_base.c +++ b/modules/serial/common/uart_base.c @@ -27,7 +27,7 @@ uint32_t uart_fbrd; uint32_t uart_baud; void enable_uart() { - register_device_memory(UART0_BASE, UART0_BASE); + register_device_memory_dmap(UART0_BASE); write32(UART0_CR, 0x0); diff --git a/modules/usb/common/dwc2.cpp b/modules/usb/common/dwc2.cpp index 22808c7e..6d3259a7 100644 --- a/modules/usb/common/dwc2.cpp +++ b/modules/usb/common/dwc2.cpp @@ -4,6 +4,7 @@ #include "memory/page_allocator.h" #include "memory/mmu.h" #include "hw/hw.h" +#include "sysregs.h" #define DWC2_INT_DATA 0b11 @@ -35,7 +36,7 @@ bool DWC2Driver::init() { use_interrupts = false; - register_device_memory(DWC2_BASE, DWC2_BASE); + register_device_memory_dmap(DWC2_BASE); dwc2 = (dwc2_regs*)DWC2_BASE; host = (dwc2_host*)(DWC2_BASE + 0x400); @@ -81,7 +82,7 @@ bool DWC2Driver::init() { setup_device(0,0); - register_device_memory(DWC2_BASE, DWC2_BASE); + register_device_memory_dmap(DWC2_BASE); return true; } @@ -95,7 +96,7 @@ uint8_t DWC2Driver::address_device(uint8_t address){ } bool DWC2Driver::make_transfer(dwc2_host_channel *channel, bool in, uint8_t pid, sizedptr data){ - channel->dma = data.ptr; + channel->dma = data.ptr ? VIRT_TO_PHYS(data.ptr) : 0; uint16_t max_size = packet_size(port_speed); uint32_t pkt_count = (data.size + max_size - 1)/max_size; channel->xfer_size = (pkt_count << 19) | (pid << 29) | data.size; @@ -204,7 +205,7 @@ bool DWC2Driver::poll(uint8_t address, uint8_t endpoint, void *out_buf, uint16_t if (endpoint_channel->cchar & 1) return false; - endpoint_channel->dma = (uintptr_t)out_buf; + endpoint_channel->dma = out_buf ? VIRT_TO_PHYS((uintptr_t)out_buf) : 0; uint16_t max_size = endpoint_channel->cchar & 0x7FF; uint32_t pkt_count = (size + max_size - 1)/max_size; diff --git a/modules/usb/common/usb.cpp b/modules/usb/common/usb.cpp index c66f9aab..db382a66 100644 --- a/modules/usb/common/usb.cpp +++ b/modules/usb/common/usb.cpp @@ -26,7 +26,7 @@ bool USBDriver::setup_device(uint8_t address, uint16_t port){ } usb_device_descriptor* descriptor = (usb_device_descriptor*)kalloc(mem_page, sizeof(usb_device_descriptor), ALIGN_64B, MEM_PRIV_KERNEL); - if (!request_descriptor(address, 0, 0x80, 6, USB_DEVICE_DESCRIPTOR, 0, 0, VIRT_TO_PHYS_P(descriptor))){ + if (!request_descriptor(address, 0, 0x80, 6, USB_DEVICE_DESCRIPTOR, 0, 0, descriptor)){ kprintf("[USB error] failed to get device descriptor"); return false; } @@ -35,7 +35,7 @@ bool USBDriver::setup_device(uint8_t address, uint16_t port){ bool use_lang_desc = true; - if (!request_descriptor(address, 0, 0x80, 6, USB_STRING_DESCRIPTOR, 0, 0, VIRT_TO_PHYS_P(lang_desc))){ + if (!request_descriptor(address, 0, 0x80, 6, USB_STRING_DESCRIPTOR, 0, 0, lang_desc)){ kprintf("[USB warning] failed to get language descriptor"); use_lang_desc = false; } @@ -49,21 +49,21 @@ bool USBDriver::setup_device(uint8_t address, uint16_t port){ //TODO: we want to maintain the strings so we can have USB device information uint16_t langid = lang_desc->lang_ids[0]; usb_string_descriptor* prod_name = (usb_string_descriptor*)kalloc(mem_page, sizeof(usb_string_descriptor), ALIGN_64B, MEM_PRIV_KERNEL); - if (request_descriptor(address, 0, 0x80, 6, USB_STRING_DESCRIPTOR, descriptor->iProduct, langid, VIRT_TO_PHYS_P(prod_name))){ + if (request_descriptor(address, 0, 0x80, 6, USB_STRING_DESCRIPTOR, descriptor->iProduct, langid, prod_name)){ char name[128]; if (utf16tochar(prod_name->unicode_string, name, sizeof(name))) { kprintf("[USB device] Product name: %s", (uint64_t)name); } } usb_string_descriptor* man_name = (usb_string_descriptor*)kalloc(mem_page, sizeof(usb_string_descriptor), ALIGN_64B, MEM_PRIV_KERNEL); - if (request_descriptor(address, 0, 0x80, 6, USB_STRING_DESCRIPTOR, descriptor->iManufacturer, langid, VIRT_TO_PHYS_P(man_name))){ + if (request_descriptor(address, 0, 0x80, 6, USB_STRING_DESCRIPTOR, descriptor->iManufacturer, langid, man_name)){ char name[128]; if (utf16tochar(man_name->unicode_string, name, sizeof(name))) { kprintf("[USB device] Manufacturer name: %s", (uint64_t)name); } } usb_string_descriptor* ser_name = (usb_string_descriptor*)kalloc(mem_page, sizeof(usb_string_descriptor), ALIGN_64B, MEM_PRIV_KERNEL); - if (request_descriptor(address, 0, 0x80, 6, USB_STRING_DESCRIPTOR, descriptor->iSerialNumber, langid, VIRT_TO_PHYS_P(ser_name))){ + if (request_descriptor(address, 0, 0x80, 6, USB_STRING_DESCRIPTOR, descriptor->iSerialNumber, langid, ser_name)){ char name[128]; if (utf16tochar(ser_name->unicode_string, name, sizeof(name))) { kprintf("[USB device] Serial: %s", (uint64_t)name); @@ -78,7 +78,7 @@ bool USBDriver::get_configuration(uint8_t address){ usb_manager->register_device(address); - usb_configuration_descriptor* config = (usb_configuration_descriptor*)VIRT_TO_PHYS_P(kalloc(mem_page, sizeof(usb_configuration_descriptor), ALIGN_64B, MEM_PRIV_KERNEL)); + usb_configuration_descriptor* config = (usb_configuration_descriptor*)kalloc(mem_page, sizeof(usb_configuration_descriptor), ALIGN_64B, MEM_PRIV_KERNEL); if (!request_sized_descriptor(address, 0, 0x80, 6, USB_CONFIGURATION_DESCRIPTOR, 0, 0, 8, config)){ kprintf("[USB error] could not get config descriptor header"); return false; diff --git a/modules/usb/common/xhci.cpp b/modules/usb/common/xhci.cpp index 7d73205f..dfe563e1 100644 --- a/modules/usb/common/xhci.cpp +++ b/modules/usb/common/xhci.cpp @@ -52,12 +52,13 @@ bool XHCIDriver::check_fatal_error() { bool XHCIDriver::init(){ uint64_t addr = 0, mmio = 0, mmio_size = 0; + uintptr_t mmio_pa = 0; bool use_pci = false; use_interrupts = true; if (XHCI_BASE){ addr = XHCI_BASE; - mmio = VIRT_TO_PHYS(addr); - register_device_memory(mmio, VIRT_TO_PHYS(mmio)); + mmio = addr; + register_device_memory_dmap(mmio); if (BOARD_TYPE == 2 && RPI_BOARD >= 5) quirk_simulate_interrupts = !pci_setup_msi_rp1(36, true); } else if (PCI_BASE) { @@ -78,14 +79,14 @@ bool XHCIDriver::init(){ pci_enable_device(addr); - if (!pci_setup_bar(addr, 0, &mmio, &mmio_size)){ + if (!pci_setup_bar(addr, 0, (uint64_t*)&mmio_pa, &mmio_size)){ kprintf("[xHCI] BARs not set up"); return false; } - pci_register(mmio, mmio_size); + pci_register(mmio_pa, mmio_size); - mmio = VIRT_TO_PHYS(mmio); + mmio = PHYS_TO_VIRT(mmio_pa); uint8_t interrupts_ok = pci_setup_interrupts(addr, INPUT_IRQ, 1); switch(interrupts_ok){ @@ -101,18 +102,16 @@ bool XHCIDriver::init(){ break; } - kprintfv("[xHCI] BARs set up @ %llx (%llx)",mmio,mmio_size); + kprintfv("[xHCI] BARs set up @ %llx (%llx)",(uint64_t)mmio_pa,mmio_size); } - mmio = VIRT_TO_PHYS(mmio); - cap = (xhci_cap_regs*)mmio; kprintfv("[xHCI] caplength %llx",cap->caplength); uintptr_t op_base = mmio + cap->caplength; - op = (xhci_op_regs*)PHYS_TO_VIRT(op_base); - ports = (xhci_port_regs*)PHYS_TO_VIRT((op_base + 0x400)); - db_base = PHYS_TO_VIRT((mmio + (cap->dboff & ~0x1F))); - rt_base = PHYS_TO_VIRT((mmio + (cap->rtsoff & ~0x1F))); + op = (xhci_op_regs*)op_base; + ports = (xhci_port_regs*)(op_base + 0x400); + db_base = mmio + (cap->dboff & ~0x1F); + rt_base = mmio + (cap->rtsoff & ~0x1F); kprintfv("[xHCI] Resetting controller %llx",&op->usbcmd); op->usbcmd &= ~1; @@ -150,7 +149,8 @@ bool XHCIDriver::init(){ mem_page = palloc(0x1000, MEM_PRIV_KERNEL, MEM_RW | MEM_DEV, false); - uintptr_t dcbaap_addr = VIRT_TO_PHYS((uintptr_t)kalloc(mem_page, (max_device_slots + 1) * sizeof(uintptr_t), ALIGN_64B, MEM_PRIV_KERNEL)); + uintptr_t* dcbaap_ptr = (uintptr_t*)kalloc(mem_page, (max_device_slots + 1) * sizeof(uintptr_t), ALIGN_64B, MEM_PRIV_KERNEL); + uintptr_t dcbaap_addr = VIRT_TO_PHYS((uintptr_t)dcbaap_ptr); op->dcbaap = dcbaap_addr; @@ -158,18 +158,22 @@ bool XHCIDriver::init(){ uint32_t scratchpad_count = ((cap->hcsparams2 >> 27) & 0x1F); - dcbaap = (uintptr_t*)PHYS_TO_VIRT(dcbaap_addr); + dcbaap = dcbaap_ptr; - uint64_t* scratchpad_array = (uint64_t*)kalloc(mem_page, (scratchpad_count == 0 ? 1 : scratchpad_count) * sizeof(uintptr_t), ALIGN_64B, MEM_PRIV_KERNEL); - for (uint32_t i = 0; i < scratchpad_count; i++) - scratchpad_array[i] = (uint64_t)kalloc(mem_page, 0x1000, ALIGN_64B, MEM_PRIV_KERNEL); - dcbaap[0] = (uint64_t)scratchpad_array; + dcbaap[0] = 0; + if (scratchpad_count) { + uint64_t* scratchpad_array = (uint64_t*)kalloc(mem_page, scratchpad_count * sizeof(uintptr_t), ALIGN_64B, MEM_PRIV_KERNEL); + uintptr_t scratchpad_array_pa = VIRT_TO_PHYS((uintptr_t)scratchpad_array); + for (uint32_t i = 0; i < scratchpad_count; i++) + scratchpad_array[i] = VIRT_TO_PHYS((uintptr_t)kalloc(mem_page, 0x1000, ALIGN_64B, MEM_PRIV_KERNEL)); + dcbaap[0] = scratchpad_array_pa; + } kprintfv("[xHCI] dcbaap assigned at %llx with %i scratchpads",dcbaap_addr,scratchpad_count); - command_ring.ring = (trb*)VIRT_TO_PHYS_P(kalloc(mem_page, MAX_TRB_AMOUNT * sizeof(trb), ALIGN_64B, MEM_PRIV_KERNEL)); + command_ring.ring = (trb*)kalloc(mem_page, MAX_TRB_AMOUNT * sizeof(trb), ALIGN_64B, MEM_PRIV_KERNEL); - op->crcr = (uintptr_t)command_ring.ring | command_ring.cycle_bit; + op->crcr = VIRT_TO_PHYS((uintptr_t)command_ring.ring) | command_ring.cycle_bit; make_ring_link(command_ring.ring, command_ring.cycle_bit); @@ -211,7 +215,7 @@ bool XHCIDriver::port_reset(uint16_t port){ kprintf("[xHCI] port %i",port); - xhci_port_regs* port_info = (xhci_port_regs*)VIRT_TO_PHYS_P(&ports[port]); + xhci_port_regs* port_info = &ports[port]; if (port_info->portsc.pp == 0){ port_info->portsc.pp = 1; @@ -253,25 +257,27 @@ bool XHCIDriver::enable_events(){ kprintfv("[xHCI] Allocating ERST"); interrupter = (xhci_interrupter*)(rt_base + 0x20); - uint64_t ev_ring = VIRT_TO_PHYS((uintptr_t)kalloc(mem_page, MAX_TRB_AMOUNT * sizeof(trb), ALIGN_64B, MEM_PRIV_KERNEL)); - uint64_t erst_addr = VIRT_TO_PHYS((uintptr_t)kalloc(mem_page, MAX_ERST_AMOUNT * sizeof(erst_entry), ALIGN_64B, MEM_PRIV_KERNEL)); - erst_entry* erst = (erst_entry*)erst_addr; + trb* ev_ring = (trb*)kalloc(mem_page, MAX_TRB_AMOUNT * sizeof(trb), ALIGN_64B, MEM_PRIV_KERNEL); + erst_entry* erst = (erst_entry*)kalloc(mem_page, MAX_ERST_AMOUNT * sizeof(erst_entry), ALIGN_64B, MEM_PRIV_KERNEL); + uint64_t ev_ring_pa = VIRT_TO_PHYS((uintptr_t)ev_ring); + uint64_t erst_pa = VIRT_TO_PHYS((uintptr_t)erst); - erst->ring_base = ev_ring; - erst->ring_size = MAX_TRB_AMOUNT; + erst[0].ring_base = ev_ring_pa; + erst[0].ring_size = MAX_TRB_AMOUNT; - kprintfv("[xHCI] ERST ring_base: %llx", ev_ring); + kprintfv("[xHCI] ERST ring_base: %llx", ev_ring_pa); kprintfv("[xHCI] ERST ring_size: %llx", erst[0].ring_size); event_ring.ring = (trb*)ev_ring; event_ring.cycle_bit = 1; + event_ring.index = 0; kprintfv("[xHCI] Interrupter register @ %llx", rt_base + 0x20); interrupter->erstsz = 1; kprintfv("[xHCI] ERSTSZ set to: %llx", (uintptr_t)interrupter->erstsz); - interrupter->erdp = ev_ring; - interrupter->erstba = erst_addr; + interrupter->erdp = ev_ring_pa; + interrupter->erstba = erst_pa; kprintfv("[xHCI] ERSTBA set to: %llx", (uintptr_t)interrupter->erstba); kprintfv("[xHCI] ERDP set to: %llx", (uintptr_t)interrupter->erdp); @@ -328,7 +334,9 @@ bool XHCIDriver::await_response(uint64_t command, uint32_t type){ uint8_t completion_code = (last_event->status >> 24) & 0xFF; if (completion_code != 1) kprintf("[xHCI error] wrong status %i on command type %llx", completion_code, ((last_event->control & TRB_TYPE_MASK) >> 10) ); - interrupter->erdp = (uintptr_t)&event_ring.ring[event_ring.index+1] | (1 << 3);//Inform of latest processed event + uint32_t erdp_index = event_ring.index + 1; + if (erdp_index >= MAX_TRB_AMOUNT) erdp_index = 0; + interrupter->erdp = VIRT_TO_PHYS((uintptr_t)&event_ring.ring[erdp_index]) | (1 << 3);//Inform of latest processed event interrupter->iman |= 1;//Clear interrupts op->usbsts |= 1 << 3;//Clear interrupts awaited_type = 0; @@ -347,7 +355,7 @@ bool XHCIDriver::issue_command(uint64_t param, uint32_t status, uint32_t control cmd->status = status; cmd->control = control | command_ring.cycle_bit; - uint64_t cmd_addr = (uintptr_t)cmd; + uint64_t cmd_addr = VIRT_TO_PHYS((uintptr_t)cmd); kprintfv("[xHCI] issuing command with control: %llx from %llx", cmd->control, cmd_addr); if (command_ring.index == MAX_TRB_AMOUNT - 1){ make_ring_link_control(command_ring.ring, command_ring.cycle_bit); @@ -376,10 +384,10 @@ bool XHCIDriver::setup_device(uint8_t address, uint16_t port){ transfer_ring->cycle_bit = 1; - xhci_input_context *ctx = (xhci_input_context*)VIRT_TO_PHYS_P(kalloc(mem_page, sizeof(xhci_input_context), ALIGN_64B, MEM_PRIV_KERNEL)); + xhci_input_context *ctx = (xhci_input_context*)kalloc(mem_page, sizeof(xhci_input_context), ALIGN_64B, MEM_PRIV_KERNEL); kprintfv("[xHCI] Allocating input context at %llx", (uintptr_t)ctx); context_map[address << 8] = (xhci_input_context *)ctx; - void* output_ctx = VIRT_TO_PHYS_P((void*)kalloc(mem_page, 0x1000, ALIGN_64B, MEM_PRIV_KERNEL)); + void* output_ctx = (void*)kalloc(mem_page, 0x1000, ALIGN_64B, MEM_PRIV_KERNEL); kprintfv("[xHCI] Allocating output for context at %llx", (uintptr_t)output_ctx); ctx->control_context.add_flags = 0b11; @@ -428,7 +436,7 @@ bool XHCIDriver::request_sized_descriptor(uint8_t address, uint8_t endpoint, uin if (descriptor_size > 0){ trb* data = &transfer_ring->ring[transfer_ring->index++]; - data->parameter = (uintptr_t)out_descriptor; + data->parameter = out_descriptor ? VIRT_TO_PHYS((uintptr_t)out_descriptor) : 0; data->status = packet.wLength; //bit 16 = direction data->control = (1 << 16) | (TRB_TYPE_DATA_STAGE << 10) | (0 << 4) | transfer_ring->cycle_bit; @@ -452,11 +460,11 @@ bool XHCIDriver::request_sized_descriptor(uint8_t address, uint8_t endpoint, uin uint8_t XHCIDriver::address_device(uint8_t address){ xhci_input_context* ctx = context_map[address << 8]; kprintfv("Addressing device %i with context %llx", address, (uintptr_t)ctx); - if (!issue_command((uintptr_t)ctx, 0, (address << 24) | (TRB_TYPE_ADDRESS_DEV << 10))){ + if (!issue_command(VIRT_TO_PHYS((uintptr_t)ctx), 0, (address << 24) | (TRB_TYPE_ADDRESS_DEV << 10))){ kprintf("[xHCI error] failed addressing device at slot %llx",address); return 0; } - xhci_device_context* context = (xhci_device_context*)dcbaap[address]; + xhci_device_context* context = (xhci_device_context*)PHYS_TO_VIRT(dcbaap[address]); kprintfv("[xHCI] ADDRESS_DEVICE %i command issued. dcbaap %llx Received packet size %i",address, (uintptr_t)dcbaap, context->endpoints[0].endpoint_f1.max_packet_size); return address; @@ -481,7 +489,7 @@ bool XHCIDriver::configure_endpoint(uint8_t address, usb_endpoint_descriptor *en kprintf("[xHCI] endpoint %i info. Direction %i type %i",ep_num, ep_dir, ep_type); xhci_input_context* ctx = context_map[address << 8]; - xhci_device_context* context = (xhci_device_context*)dcbaap[address]; + xhci_device_context* context = (xhci_device_context*)PHYS_TO_VIRT(dcbaap[address]); ctx->control_context.add_flags = (1 << 0) | (1 << ep_num); if (ep_num > ctx->device_context.slot_f0.context_entries) @@ -503,7 +511,7 @@ bool XHCIDriver::configure_endpoint(uint8_t address, usb_endpoint_descriptor *en ctx->device_context.endpoints[ep_num-1].endpoint_f23.ring_ptr = VIRT_TO_PHYS((uintptr_t)ep_ring->ring) >> 4; ctx->device_context.endpoints[ep_num-1].endpoint_f4.average_trb_length = sizeof(trb); - if (!issue_command((uintptr_t)ctx, 0, (address << 24) | (TRB_TYPE_CONFIG_EP << 10))){ + if (!issue_command(VIRT_TO_PHYS((uintptr_t)ctx), 0, (address << 24) | (TRB_TYPE_CONFIG_EP << 10))){ kprintf("[xHCI] Failed to configure endpoint %i for address %i",ep_num,address); return false; } @@ -538,11 +546,11 @@ bool XHCIDriver::poll(uint8_t address, uint8_t endpoint, void *out_buf, uint16_t } void XHCIDriver::handle_interrupt(){ - trb* ev = (trb*)PHYS_TO_VIRT_P(&event_ring.ring[event_ring.index]); + trb* ev = &event_ring.ring[event_ring.index]; if (!((ev->control & 1) == event_ring.cycle_bit)) return; uint32_t type = (ev->control & TRB_TYPE_MASK) >> 10; uint64_t addr = ev->parameter; - if (type == awaited_type && (awaited_addr == 0 || (awaited_addr & 0xFFFFFFFF) == addr)) + if (type == awaited_type && (awaited_addr == 0 || awaited_addr == addr)) return; kprintfv("[xHCI] >>> Unhandled interrupt %i %llx",event_ring.index,type); uint8_t completion_code = (ev->status >> 24) & 0xFF; diff --git a/run_virt b/run_virt index c6804aa8..56250871 100755 --- a/run_virt +++ b/run_virt @@ -3,8 +3,17 @@ echo "Running virt emulator" ARGS="" +SERIAL_OPT="-serial stdio" if [ "$1" = "debug" ]; then ARGS="-monitor unix:/tmp/qemu-monitor-socket,server,nowait -s -S" + + if [ -r /etc/os-release ]; then + . /etc/os-release + if [ "${ID:-}" = "ubuntu" ]; then + : > /tmp/serial.log 2>/dev/null || true + SERIAL_OPT="-serial file:/tmp/serial.log" + fi + fi fi MSI_CAPABILITIES="" @@ -22,8 +31,8 @@ AUDIO_BACKEND="sdl" VIRTIO_GPU_VARS=",xres=1920,yres=1080" GL="on" -USE_FB=true -USE_NET=false +USE_FB=false +USE_NET=true if [ "$USE_FB" == true ]; then SELECTED_GPU="ramfb" @@ -78,7 +87,7 @@ fi echo "Using networking mode: $NETDEV $NETDRIV $DUMP" -$PRIVILEGE qemu-system-aarch64 \ +exec $PRIVILEGE qemu-system-aarch64 \ -M virt \ -cpu cortex-a72 \ -m 512M \ @@ -88,7 +97,7 @@ $PRIVILEGE qemu-system-aarch64 \ $NETDEV \ $NETDRIV \ $DUMP \ - -serial stdio \ + $SERIAL_OPT \ -drive file=disk.img,if=none,format=raw,id=hd0 \ -device virtio-blk-pci,drive=hd0 \ $SHAREDFS \ diff --git a/rundebug b/rundebug index 7503a439..e91dab15 100755 --- a/rundebug +++ b/rundebug @@ -12,13 +12,24 @@ done echo $OS_TYPE -if [[ "$(uname)" = "Darwin" ]]; then +if [[ "$(uname)" == "Darwin" ]]; then osascript </dev/null 2>&1 & + + exec ./debug "${ARGS[@]}" else - (./run_$MODE debug) & (./debug ${ARGS[*]}) + (./run_$MODE debug) & (exec ./debug "${ARGS[@]}") fi diff --git a/shared b/shared index e988bcac..86fc342d 160000 --- a/shared +++ b/shared @@ -1 +1 @@ -Subproject commit e988bcacedd8fb329322e4ccdb614912c4d1eafc +Subproject commit 86fc342df95a45daac6695a25ff6e77c29885323 diff --git a/user/linker.ld b/user/linker.ld index 96f61965..3060b9cd 100644 --- a/user/linker.ld +++ b/user/linker.ld @@ -1,25 +1,30 @@ ENTRY(main) +PHDRS { + text PT_LOAD FLAGS(5); + data PT_LOAD FLAGS(6); +} SECTIONS { . = 0x1000; .text : { *(.text .text.*) - } + }:text .rodata : { *(.rodata .rodata.*) - } + } :text + . = ALIGN(0x1000); + + .data : { + *(.data .data.*) + } :data - .bss : { + .bss (NOLOAD) : { __bss_start = .; *(.bss .bss.* COMMON) __bss_end = .; - } - - .data : { - *(.data .data.*) - } + } :data /DISCARD/ : { *(.comment .note .eh_frame) diff --git a/utils/terminal/linker.ld b/utils/terminal/linker.ld index 055d18fd..3060b9cd 100644 --- a/utils/terminal/linker.ld +++ b/utils/terminal/linker.ld @@ -1,25 +1,30 @@ ENTRY(main) +PHDRS { + text PT_LOAD FLAGS(5); + data PT_LOAD FLAGS(6); +} SECTIONS { . = 0x1000; .text : { *(.text .text.*) - } + }:text .rodata : { *(.rodata .rodata.*) - } + } :text + . = ALIGN(0x1000); .data : { *(.data .data.*) - } + } :data - .bss : { + .bss (NOLOAD) : { __bss_start = .; *(.bss .bss.* COMMON) __bss_end = .; - } + } :data /DISCARD/ : { *(.comment .note .eh_frame) diff --git a/utils/terminal/terminal.cpp b/utils/terminal/terminal.cpp index 7abf533e..3f897b5c 100644 --- a/utils/terminal/terminal.cpp +++ b/utils/terminal/terminal.cpp @@ -2,14 +2,19 @@ #include "alloc/allocate.h" #include "std/std.h" #include "input_keycodes.h" - +//TODO why do ping and tracert often block the terminal even if their process finishes? Terminal::Terminal() : Console() { uint32_t color_buf[2] = {}; sreadf("/theme", &color_buf, sizeof(uint64_t)); default_bg_color = color_buf[0]; bg_color = color_buf[0]; default_text_color = color_buf[1]; - text_color = color_buf[1]; + if ((default_bg_color & 0xFF000000) == 0) default_bg_color |= 0xFF000000; + if ((default_text_color & 0xFF000000) == 0) default_text_color |= 0xFF000000; + + bg_color = default_bg_color; + text_color = default_text_color; + if (text_color == bg_color) text_color = default_text_color = 0xFFFFFFFF; char_scale = 2; prompt_length = 2; @@ -169,9 +174,9 @@ bool Terminal::exec_cmd(const char *cmd, int argc, const char *argv[]){ file out_fd, state_fd; openf(s1.data, &out_fd); - free_sized(s1.data, s1.mem_length); + string_free(s1); openf(s2.data, &state_fd); - free_sized(s2.data, s2.mem_length); + string_free(s2); int state = 1; size_t amount = 0x100; @@ -197,13 +202,13 @@ bool Terminal::exec_cmd(const char *cmd, int argc, const char *argv[]){ put_string(buf); } - free_sized(buf, amount + 1); + release(buf); closef(&out_fd); closef(&state_fd); string exit_msg = string_format("\nProcess %i ended.", proc); put_string(exit_msg.data); - free_sized(exit_msg.data, exit_msg.mem_length); + string_free(exit_msg); return true; } @@ -229,13 +234,13 @@ const char** Terminal::parse_arguments(char *args, int *count){ void Terminal::run_command(){ if (input_len) { if (history_len == history_max) { - if (history[0]) free_sized(history[0], strlen(history[0]) + 1); + if (history[0]) release(history[0]); for (uint32_t i = 1; i < history_max; i++) history[i - 1] = history[i]; history_len = history_max - 1; } uint32_t n = input_len; - char *copy = (char*)malloc(n + 1); + char *copy = (char*)zalloc(n + 1); if (copy) { memcpy(copy, input_buf, n); copy[n] = 0; @@ -286,8 +291,8 @@ void Terminal::run_command(){ } if (argv) release((void*)argv); - free_sized(cmd.data, cmd.mem_length); - if (args_copy.mem_length) free_sized(args_copy.data, args_copy.mem_length); + string_free(cmd); + string_free(args_copy); command_running = true; } @@ -378,8 +383,9 @@ bool Terminal::handle_input(){ } draw_ctx* Terminal::get_ctx(){ - if (dctx) free_sized(dctx, sizeof(draw_ctx)); - draw_ctx *ctx = (draw_ctx*)malloc(sizeof(draw_ctx)); + if (dctx) release(dctx); + draw_ctx *ctx = (draw_ctx*)zalloc(sizeof(draw_ctx)); + if (!ctx) return nullptr; request_draw_ctx(ctx); return ctx; }