meta: rename kernel to Magenta

whether a mapping is private or shared determines how the mapping is handled
when a task is duplicated.

CMakeLists.txt

@@ -1,5 +1,5 @@
-cmake_minimum_required(VERSION 3.13)
+cmake_minimum_required(VERSION 3.31)
-project(mango C ASM)
+project(magenta C ASM)
 
 if (NOT BUILD_TOOLS_DIR)
 	message(FATAL_ERROR "No build tools directory specified. Please run build.sh")
@@ -9,8 +9,8 @@ set(CMAKE_C_STANDARD 17)
 
 set(kernel_arch x86_64)
 
-set(kernel_name "Mango")
+set(kernel_name "Magenta")
-set(kernel_exe_name "mango_kernel")
+set(kernel_exe_name "magenta_kernel")
 
 set(generic_src_dirs ds init kernel libc sched util vm syscall)
 set(kernel_sources "")
@@ -28,8 +28,6 @@ file(GLOB_RECURSE arch_sources_c arch/${kernel_arch}/*.c)
 file(GLOB_RECURSE arch_sources_asm arch/${kernel_arch}/*.S)
 file(GLOB_RECURSE arch_headers arch/${kernel_arch}/*.h)
 
-set_property(SOURCE ${arch_sources_asm} PROPERTY LANGUAGE C)
-
 add_executable(${kernel_exe_name}
 	${kernel_sources}
 	${kernel_headers}
@@ -40,6 +38,7 @@ add_executable(${kernel_exe_name}
 target_include_directories(${kernel_exe_name} PRIVATE
 	include
 	libc/include
+	libmagenta/include
 	arch/${kernel_arch}/include)
 target_compile_options(${kernel_exe_name} PRIVATE
 	-nostdlib -ffreestanding

README

@@ -1,4 +1,4 @@
-Mango
+Magenta
 =====
 
 It's a kernel!

arch/user/cpu.c

@@ -1,5 +1,5 @@
 #include <unistd.h>
-#include <mango/machine/cpu.h>
+#include <kernel/machine/cpu.h>
 
 int ml_init_bootcpu(void)
 {

arch/user/hwlock.c

@@ -1,5 +1,5 @@
-#include <mango/machine/hwlock.h>
+#include <kernel/compiler.h>
-#include <mango/compiler.h>
+#include <kernel/machine/hwlock.h>
 
 void ml_hwlock_lock(ml_hwlock_t *lck)
 {

arch/user/include/mango/machine/cpu.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_USER_CPU_H_
+#ifndef MAGENTA_USER_CPU_H_
-#define MANGO_USER_CPU_H_
+#define MAGENTA_USER_CPU_H_
 
 #ifdef __cplusplus
 extern "C" {

arch/user/include/mango/machine/hwlock.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_USER_HWLOCK_H_
+#ifndef MAGENTA_USER_HWLOCK_H_
-#define MANGO_USER_HWLOCK_H_
+#define MAGENTA_USER_HWLOCK_H_
 
 #define ML_HWLOCK_INIT (0)
 

arch/user/include/mango/machine/init.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_INIT_H_
+#ifndef MAGENTA_X86_64_INIT_H_
-#define MANGO_X86_64_INIT_H_
+#define MAGENTA_X86_64_INIT_H_
 
 #include <stddef.h>
 #include <stdint.h>

arch/user/include/mango/machine/irq.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_IRQ_H_
+#ifndef MAGENTA_X86_64_IRQ_H_
-#define MANGO_X86_64_IRQ_H_
+#define MAGENTA_X86_64_IRQ_H_
 
 
 #endif

arch/user/include/mango/machine/pmap.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_USER_PMAP_H_
+#ifndef MAGENTA_USER_PMAP_H_
-#define MANGO_USER_PMAP_H_
+#define MAGENTA_USER_PMAP_H_
 
 #include <stdint.h>
 

arch/user/include/mango/machine/vm.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_USER_VM_H_
+#ifndef MAGENTA_USER_VM_H_
-#define MANGO_USER_VM_H_
+#define MAGENTA_USER_VM_H_
 
 #include <stdint.h>
 

arch/user/init.c

@@ -1,11 +1,11 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <mango/init.h>
+#include <kernel/init.h>
-#include <mango/memblock.h>
+#include <kernel/memblock.h>
-#include <mango/vm.h>
+#include <kernel/vm.h>
-#include <mango/object.h>
+#include <kernel/object.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
 #include <arch/stdcon.h>
 #include <sys/mman.h>
 

arch/user/initcall.c

@@ -1,4 +1,4 @@
-#include <mango/init.h>
+#include <kernel/init.h>
 
 #ifdef __APPLE__
 extern char __start_initcall0[] __asm("section$start$__DATA$__initcall0.init");

arch/user/stdcon.c

@@ -1,10 +1,10 @@
-#include <mango/libc/string.h>
+#include <kernel/libc/string.h>
-#include <mango/libc/ctype.h>
+#include <kernel/libc/ctype.h>
 #include <stdint.h>
 #include <stdio.h>
-#include <mango/console.h>
+#include <kernel/console.h>
-#include <mango/vm.h>
+#include <kernel/vm.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
 
 static void stdcon_write(struct console *con, const char *s, unsigned int len)
 {

arch/x86_64/asm-offset.c

@@ -1,10 +1,11 @@
-#include <mango/sched.h>
+#include <kernel/compiler.h>
-#include <mango/compiler.h>
+#include <kernel/sched.h>
+#include <kernel/thread.h>
 
 // size_t THREAD_sp = offsetof(struct thread, tr_sp);
 
-/* Use %a0 instead of %0 to prevent gcc from emitting a $ before the symbol value
+/* Use %a0 instead of %0 to prevent gcc from emitting a $ before the symbol
-   in the generated assembly.
+   value in the generated assembly.
 
    emitting
      .set TASK_sp, $56
@@ -16,12 +17,17 @@
 */
 
 #define DEFINE(sym, val)                                                       \
-	asm volatile("\n.global " #sym "\n.type " #sym ", @object" "\n.set " #sym ", %a0" : : "i" (val))
+	asm volatile("\n.global " #sym "\n.type " #sym                         \
+	             ", @object"                                               \
+	             "\n.set " #sym ", %a0"                                    \
+	             :                                                         \
+	             : "i"(val))
 
-#define OFFSET(sym, str, mem) \
+#define OFFSET(sym, str, mem) DEFINE(sym, offsetof(str, mem))
-	DEFINE(sym, offsetof(str, mem))
 
 static void __used common(void)
 {
 	OFFSET(THREAD_sp, struct thread, tr_sp);
+	OFFSET(THREAD_fsbase, struct thread, tr_ml.tr_fsbase);
+	OFFSET(THREAD_gsbase, struct thread, tr_ml.tr_gsbase);
 }

arch/x86_64/config.cmake

@@ -1,10 +1,5 @@
 target_compile_options(${kernel_exe_name} PRIVATE
 	-z max-page-size=0x1000 -m64 -mcmodel=large -mno-red-zone -mno-mmx
 	-mno-sse -mno-sse2 -D_64BIT -DBYTE_ORDER=1234)
-target_link_libraries(${kernel_exe_name} "-z max-page-size=0x1000" "-T ${CMAKE_CURRENT_SOURCE_DIR}/arch/x86_64/layout.ld")
+target_link_libraries(${kernel_exe_name} "-static -z max-page-size=0x1000" "-T ${CMAKE_CURRENT_SOURCE_DIR}/arch/x86_64/layout.ld")
 
-add_custom_command(TARGET ${kernel_exe_name} POST_BUILD
-	COMMAND ${BUILD_TOOLS_DIR}/e64patch $<TARGET_FILE:${kernel_exe_name}>
-	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-	COMMENT "Patching kernel elf64 image"
-)

arch/x86_64/cpu.c

@@ -1,5 +1,5 @@
 #include <arch/msr.h>
-#include <mango/machine/cpu.h>
+#include <kernel/machine/cpu.h>
 
 int ml_cpu_block_init(ml_cpu_block *p)
 {

arch/x86_64/e820.c

@@ -1,8 +1,8 @@
-#include "mango/types.h"
-#include <mango/memblock.h>
-#include <mango/printk.h>
-#include <mango/util.h>
 #include <arch/e820.h>
+#include <kernel/memblock.h>
+#include <kernel/printk.h>
+#include <kernel/types.h>
+#include <kernel/util.h>
 
 void e820_scan(multiboot_memory_map_t *mmap, size_t len)
 {
@@ -36,7 +36,9 @@ void e820_scan(multiboot_memory_map_t *mmap, size_t len)
 		}
 
 		printk("e820: [mem 0x%016llx-0x%016llx] %s",
-		       entry->addr, entry->addr + entry->len - 1, type);
+		       entry->addr,
+		       entry->addr + entry->len - 1,
+		       type);
 
 		memblock_add(entry->addr, entry->len);
 
@@ -53,7 +55,12 @@ void e820_scan(multiboot_memory_map_t *mmap, size_t len)
 
 	char str_mem_total[64], str_mem_reserved[64];
 	data_size_to_string(mem_total, str_mem_total, sizeof str_mem_total);
-	data_size_to_string(mem_reserved, str_mem_reserved, sizeof str_mem_reserved);
+	data_size_to_string(
+		mem_reserved,
+		str_mem_reserved,
+		sizeof str_mem_reserved);
 
-	printk("e820: total memory: %s, hw reserved: %s", str_mem_total, str_mem_reserved);
+	printk("e820: total memory: %s, hw reserved: %s",
+	       str_mem_total,
+	       str_mem_reserved);
 }

arch/x86_64/gdt.c

@@ -1,7 +1,7 @@
 #include <arch/gdt.h>
 #include <arch/tss.h>
-#include <mango/libc/string.h>
+#include <kernel/libc/string.h>
-#include <mango/types.h>
+#include <kernel/types.h>
 #include <stddef.h>
 
 static void init_entry(struct gdt_entry *entry, int access, int flags)
@@ -28,11 +28,11 @@ int gdt_init(struct gdt *gdt, struct gdt_ptr *gdtp)
 		GDT_F_64BIT);
 	init_entry(
 		&gdt->g_entries[3],
-		GDT_A_PRESENT | GDT_A_USER | GDT_A_CODEREAD | GDT_A_CODE,
+		GDT_A_PRESENT | GDT_A_USER | GDT_A_DATAWRITE | GDT_A_DATA,
 		GDT_F_64BIT);
 	init_entry(
 		&gdt->g_entries[4],
-		GDT_A_PRESENT | GDT_A_USER | GDT_A_DATAWRITE | GDT_A_DATA,
+		GDT_A_PRESENT | GDT_A_USER | GDT_A_CODEREAD | GDT_A_CODE,
 		GDT_F_64BIT);
 
 	gdtp->g_ptr = (uint64_t)gdt;

arch/x86_64/hwlock.S

@@ -11,6 +11,41 @@ ml_hwlock_lock:
 
 	mov $1, %ecx
 
+	mfence
+
+1:	mov $0, %eax
+	lock cmpxchg %ecx, (%rdi)
+	jne 1b
+
+	pop %rbp
+	ret
+
+
+	.global ml_hwlock_unlock
+	.type ml_hwlock_unlock, @function
+
+	/* %rdi = pointer to ml_hwlock_t (int) */
+ml_hwlock_unlock:
+	push %rbp
+	mov %rsp, %rbp
+
+	movl $0, (%rdi)
+	mfence
+
+	pop %rbp
+	ret
+
+
+	.global ml_hwlock_lock_irq
+	.type ml_hwlock_lock_irq, @function
+
+	/* %rdi = pointer to ml_hwlock_t (int) */
+ml_hwlock_lock_irq:
+	push %rbp
+	mov %rsp, %rbp
+
+	mov $1, %ecx
+
 	cli
 	mfence
 
@@ -21,11 +56,12 @@ ml_hwlock_lock:
 	pop %rbp
 	ret
 
-	.global ml_hwlock_unlock
+
-	.type ml_hwlock_unlock, @function
+	.global ml_hwlock_unlock_irq
+	.type ml_hwlock_unlock_irq, @function
 
 	/* %rdi = pointer to ml_hwlock_t (int) */
-ml_hwlock_unlock:
+ml_hwlock_unlock_irq:
 	push %rbp
 	mov %rsp, %rbp
 
@@ -62,6 +98,7 @@ ml_hwlock_lock_irqsave:
 	pop %rbp
 	ret
 
+
 	.global ml_hwlock_unlock_irqrestore
 	.type ml_hwlock_unlock_irqrestore, @function
 

arch/x86_64/include/arch/gdt.h

@@ -2,7 +2,7 @@
 #define ARCH_GDT_H_
 
 #include <arch/tss.h>
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
 #include <stdint.h>
 
 #ifdef __cplusplus

arch/x86_64/include/arch/irq.h

@@ -1,8 +1,8 @@
 #ifndef ARCH_IRQ_H_
 #define ARCH_IRQ_H_
 
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
-#include <mango/queue.h>
+#include <kernel/queue.h>
 #include <stdint.h>
 
 #ifdef __cplusplus

arch/x86_64/include/arch/msr.h

@@ -3,7 +3,9 @@
 
 #include <stdint.h>
 
+#define MSR_FS_BASE        0xC0000100
 #define MSR_GS_BASE        0xC0000101
+#define MSR_KERNEL_GS_BASE 0xC0000102
 
 #ifdef __cplusplus
 extern "C" {

arch/x86_64/include/arch/paging.h

@@ -1,8 +1,8 @@
 #ifndef ARCH_PAGING_H_
 #define ARCH_PAGING_H_
 
-#include <mango/types.h>
+#include <kernel/compiler.h>
-#include <mango/compiler.h>
+#include <kernel/types.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -55,6 +55,7 @@ enum page_size {
    defined in pmap_ctrl.S */
 extern int gigabyte_pages(void);
 extern int enable_nx(void);
+extern int enable_wp(void);
 
 #ifdef __cplusplus
 }

arch/x86_64/include/arch/tss.h

@@ -1,8 +1,8 @@
 #ifndef ARCH_TSS_H_
 #define ARCH_TSS_H_
 
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
-#include <mango/types.h>
+#include <kernel/types.h>
 #include <stdint.h>
 
 #define TSS_GDT_INDEX 5

arch/x86_64/include/kernel/machine/cpu.h

@@ -1,10 +1,10 @@
-#ifndef MANGO_X86_64_CPU_H_
+#ifndef KERNEL_X86_64_CPU_H_
-#define MANGO_X86_64_CPU_H_
+#define KERNEL_X86_64_CPU_H_
 
 #include <arch/gdt.h>
 #include <arch/irq.h>
 #include <arch/tss.h>
-#include <mango/types.h>
+#include <kernel/types.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -38,6 +38,10 @@ typedef struct ml_cpu_block {
 	struct cpu_data *c_data;
 } ml_cpu_block;
 
+struct ml_int_context {
+	uint64_t rip, cs, rflags, rsp, ss;
+};
+
 struct ml_cpu_context {
 	uint64_t r15, r14, r13, r12, r11, r10, r9, r8;
 	uint64_t rdi, rsi, rbp, unused_rsp, rbx, rdx, rcx, rax;

arch/x86_64/include/kernel/machine/hwlock.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_HWLOCK_H_
+#ifndef KERNEL_X86_64_HWLOCK_H_
-#define MANGO_X86_64_HWLOCK_H_
+#define KERNEL_X86_64_HWLOCK_H_
 
 #define ML_HWLOCK_INIT (0)
 
@@ -12,6 +12,9 @@ typedef int ml_hwlock_t;
 extern void ml_hwlock_lock(ml_hwlock_t *lck);
 extern void ml_hwlock_unlock(ml_hwlock_t *lck);
 
+extern void ml_hwlock_lock_irq(ml_hwlock_t *lck);
+extern void ml_hwlock_unlock_irq(ml_hwlock_t *lck);
+
 extern void ml_hwlock_lock_irqsave(ml_hwlock_t *lck, unsigned long *flags);
 extern void ml_hwlock_unlock_irqrestore(ml_hwlock_t *lck, unsigned long flags);
 

arch/x86_64/include/kernel/machine/init.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_INIT_H_
+#ifndef KERNEL_X86_64_INIT_H_
-#define MANGO_X86_64_INIT_H_
+#define KERNEL_X86_64_INIT_H_
 
 #include <stddef.h>
 #include <stdint.h>

arch/x86_64/include/kernel/machine/irq.h

@@ -0,0 +1,5 @@
+#ifndef KERNEL_X86_64_IRQ_H_
+#define KERNEL_X86_64_IRQ_H_
+
+
+#endif

arch/x86_64/include/kernel/machine/panic.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_PANIC_H_
+#ifndef KERNEL_X86_64_PANIC_H_
-#define MANGO_X86_64_PANIC_H_
+#define KERNEL_X86_64_PANIC_H_
 
 #include <stdint.h>
 

arch/x86_64/include/kernel/machine/pmap.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_PMAP_H_
+#ifndef KERNEL_X86_64_PMAP_H_
-#define MANGO_X86_64_PMAP_H_
+#define KERNEL_X86_64_PMAP_H_
 
 #include <arch/paging.h>
 

arch/x86_64/include/kernel/machine/random.h

@@ -0,0 +1,10 @@
+#ifndef KERNEL_X86_64_RANDOM_H_
+#define KERNEL_X86_64_RANDOM_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+
+extern bool ml_hwrng_available(void);
+extern uint64_t ml_hwrng_generate(void);
+
+#endif

arch/x86_64/include/kernel/machine/thread.h

@@ -1,7 +1,11 @@
-#ifndef MANGO_X86_64_THREAD_H_
+#ifndef KERNEL_X86_64_THREAD_H_
-#define MANGO_X86_64_THREAD_H_
+#define KERNEL_X86_64_THREAD_H_
 
-#include <mango/sched.h>
+#include <kernel/sched.h>
+
+struct ml_thread {
+	virt_addr_t tr_gsbase, tr_fsbase;
+};
 
 struct ml_cpu_context;
 
@@ -21,9 +25,30 @@ extern void ml_thread_switch_user(void);
 extern void ml_thread_prepare_kernel_context(uintptr_t ip, uintptr_t *sp);
 /* prepare the stack so that ml_thread_switch_user can jump to usermode
  * with the specified IP/user SP */
-extern void ml_thread_prepare_user_context(
+extern kern_status_t ml_thread_prepare_user_context(
 	virt_addr_t ip,
 	virt_addr_t user_sp,
+	virt_addr_t *kernel_sp,
+	const uintptr_t *args,
+	size_t nr_args);
+/* prepare the stack so that ml_thread_switch_user can jump to usermode
+ * with the specified register context */
+extern kern_status_t ml_thread_clone_user_context(
+	const struct ml_cpu_context *src_regs,
+	const struct ml_thread *src_ml,
+	struct ml_thread *dest_ml,
+	uintptr_t return_value,
 	virt_addr_t *kernel_sp);
 
+extern kern_status_t ml_thread_config_get(
+	struct thread *thread,
+	kern_config_key_t key,
+	void *out,
+	size_t max);
+extern kern_status_t ml_thread_config_set(
+	struct thread *thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+
 #endif

arch/x86_64/include/kernel/machine/vm.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_VM_H_
+#ifndef KERNEL_X86_64_VM_H_
-#define MANGO_X86_64_VM_H_
+#define KERNEL_X86_64_VM_H_
 
 /* kernel higher-half base virtual address. */
 #define VM_KERNEL_VOFFSET 0xFFFFFFFF80000000

arch/x86_64/include/mango/machine/irq.h

@@ -1,5 +0,0 @@
-#ifndef MANGO_X86_64_IRQ_H_
-#define MANGO_X86_64_IRQ_H_
-
-
-#endif

arch/x86_64/init.c

@@ -2,24 +2,41 @@
 #include <arch/pit.h>
 #include <arch/serial.h>
 #include <arch/vgacon.h>
-#include <mango/arg.h>
+#include <kernel/arg.h>
-#include <mango/bsp.h>
+#include <kernel/bsp.h>
-#include <mango/clock.h>
+#include <kernel/clock.h>
-#include <mango/console.h>
+#include <kernel/console.h>
-#include <mango/cpu.h>
+#include <kernel/cpu.h>
-#include <mango/init.h>
+#include <kernel/init.h>
-#include <mango/libc/stdio.h>
+#include <kernel/libc/stdio.h>
-#include <mango/machine/cpu.h>
+#include <kernel/machine/cpu.h>
-#include <mango/memblock.h>
+#include <kernel/machine/random.h>
-#include <mango/object.h>
+#include <kernel/memblock.h>
-#include <mango/percpu.h>
+#include <kernel/object.h>
-#include <mango/pmap.h>
+#include <kernel/percpu.h>
-#include <mango/printk.h>
+#include <kernel/pmap.h>
-#include <mango/types.h>
+#include <kernel/printk.h>
-#include <mango/vm.h>
+#include <kernel/types.h>
+#include <kernel/util.h>
+#include <kernel/vm.h>
+
+#define HARDWARE_RNG
 
 #define PTR32(x)            ((void *)((uintptr_t)(x)))
 
+/* the physical address of the start of the memblock heap.
+ * this is an arbirary value; the heap can start anywhere in memory.
+ * any reserved areas of memory (the kernel, bsp, bios data, etc) are
+ * automatically taken into account.
+ * HOWEVER, this value will dictate how much physical memory is required for
+ * the kernel to boot successfully.
+ * the value of 16MiB (0x1000000) means that all heap allocations will be
+ * above 16MiB, leaving the area below free for DMA operations.
+ * this value CAN be reduced all the way to zero to minimise the amount of
+ * memory required to boot, but this may leave you with no DMA memory available.
+ */
+#define MEMBLOCK_HEAP_START 0x1000000
+
 static ml_cpu_block g_bootstrap_cpu = {0};
 
 /* start and end of kernel image (physical addresses) */
@@ -33,7 +50,7 @@ static void bootstrap_cpu_init(void)
 
 static void early_vm_init(uintptr_t reserve_end)
 {
-	uintptr_t alloc_start = VM_KERNEL_VOFFSET;
+	uintptr_t alloc_start = VM_KERNEL_VOFFSET + MEMBLOCK_HEAP_START;
 	/* boot code mapped 2 GiB of memory from
 	   VM_KERNEL_VOFFSET */
 	uintptr_t alloc_end = VM_KERNEL_VOFFSET + 0x7fffffff;
@@ -110,6 +127,22 @@ int ml_init(uintptr_t arg)
 		reserve_end = bsp.mod_base + bsp.mod_size;
 	}
 
+#if defined(HARDWARE_RNG)
+	if (ml_hwrng_available()) {
+		printk("cpu: ardware RNG available");
+		uint64_t seed = ml_hwrng_generate();
+		printk("cpu: RNG seed=%zx", seed);
+		init_random(seed);
+	} else {
+#endif
+		printk("cpu: hardware RNG unavailable");
+		uint64_t seed = 0xeddc4c8a679dc23f;
+		printk("cpu: RNG seed=%zx", seed);
+		init_random(seed);
+#if defined(HARDWARE_RNG)
+	}
+#endif
+
 	early_vm_init(reserve_end);
 
 	e820_scan(PTR32(mb->mmap_addr), mb->mmap_length);

arch/x86_64/initcall.c

@@ -1,4 +1,4 @@
-#include <mango/init.h>
+#include <kernel/init.h>
 
 extern char __initcall0_start[];
 extern char __initcall1_start[];

arch/x86_64/irq.c

@@ -1,13 +1,14 @@
 #include <arch/irq.h>
 #include <arch/msr.h>
 #include <arch/ports.h>
-#include <mango/cpu.h>
+#include <kernel/cpu.h>
-#include <mango/libc/string.h>
+#include <kernel/libc/string.h>
-#include <mango/machine/cpu.h>
+#include <kernel/machine/cpu.h>
-#include <mango/machine/irq.h>
+#include <kernel/machine/irq.h>
-#include <mango/panic.h>
+#include <kernel/panic.h>
-#include <mango/sched.h>
+#include <kernel/sched.h>
-#include <mango/syscall.h>
+#include <kernel/syscall.h>
+#include <kernel/thread.h>
 #include <stddef.h>
 
 #define MAX_ISR_HANDLERS  16
@@ -31,7 +32,7 @@ static uintptr_t int_entry_points[NR_IDT_ENTRIES];
 static void set_syscall_gate(uintptr_t rip)
 {
 	uint64_t user_cs = 0x13;
-	uint64_t kernel_cs = 0x8;
+	uint64_t kernel_cs = 0x08;
 
 	uintptr_t star_reg = 0xC0000081;
 	uintptr_t lstar_reg = 0xC0000082;
@@ -97,11 +98,9 @@ static void pf_handler(struct ml_cpu_context *regs)
 
 	virt_addr_t fault_ptr = pf_faultptr();
 
-	kern_status_t status = KERN_FATAL_ERROR;
+	ml_int_enable();
-
+	kern_status_t status = pmap_handle_fault(fault_ptr, fault_flags);
-	if (regs->err_no & PF_USER) {
+	ml_int_disable();
-		status = pmap_handle_fault(fault_ptr, fault_flags);
-	}
 
 	if (status == KERN_OK) {
 		return;
@@ -168,6 +167,12 @@ int idt_load(struct idt_ptr *ptr)
 
 void isr_dispatch(struct ml_cpu_context *regs)
 {
+	struct thread *thr = get_current_thread();
+	if (thr) {
+		thr->tr_irqctx = regs;
+		put_current_thread(thr);
+	}
+
 	int_hook h = isr_handlers[regs->int_no];
 	if (h) {
 		h(regs);
@@ -190,6 +195,13 @@ void irq_dispatch(struct ml_cpu_context *regs)
 	end_charge_period();
 
 	irq_ack(regs->int_no);
+
+	struct thread *thr = get_current_thread();
+	if (thr) {
+		thr->tr_irqctx = regs;
+		put_current_thread(thr);
+	}
+
 	struct queue *hooks = &irq_hooks[regs->int_no - IRQ0];
 	queue_foreach(struct irq_hook, hook, hooks, irq_entry)
 	{
@@ -205,6 +217,12 @@ void irq_dispatch(struct ml_cpu_context *regs)
 
 void syscall_dispatch(struct ml_cpu_context *regs)
 {
+	struct thread *thr = get_current_thread();
+	if (thr) {
+		thr->tr_irqctx = regs;
+		put_current_thread(thr);
+	}
+
 	unsigned int sysid = regs->rax;
 	virt_addr_t syscall_impl = syscall_get_function(sysid);
 
@@ -226,6 +244,7 @@ void syscall_dispatch(struct ml_cpu_context *regs)
 
 	SYSCALL_SIGNATURE(fn) = (SYSCALL_SIGNATURE())syscall_impl;
 
+	ml_int_enable();
 	regs->rax
 		= fn(regs->rdi,
 	             regs->rsi,
@@ -235,6 +254,7 @@ void syscall_dispatch(struct ml_cpu_context *regs)
 	             regs->r9,
 	             regs->r13,
 	             regs->r14);
+	ml_int_disable();
 }
 
 void hook_irq(enum irq_vector vec, struct irq_hook *hook)

arch/x86_64/irqvec.S

@@ -333,11 +333,30 @@ IRQ 223, 255
 isr_common_stub:
 	PUSH_REGS
 
+	# When ISR occurs in Ring 3, CPU sets %ss (and other non-code selectors)
+	# to 0.
+	mov %ss, %ax
+	cmp $0, %ax
+	jne isr_skipgs1
+
+	mov $0x10, %ax
+	mov %ax, %ss
+	swapgs
+
+isr_skipgs1:
 	mov %rsp, %rdi
 	call isr_dispatch
 
 	POP_REGS
 	add $16, %rsp
+
+	cmpq $0x1b, 32(%rsp)
+	jne isr_skipgs2
+
+	swapgs
+
+isr_skipgs2:
+
 	iretq
 
 
@@ -347,11 +366,31 @@ isr_common_stub:
 irq_common_stub:
 	PUSH_REGS
 
+	# When IRQ occurs in Ring 3, CPU sets %ss (and other non-code selectors)
+	# to 0.
+	mov %ss, %ax
+	cmp $0, %ax
+	jne irq_skipgs1
+
+	mov $0x10, %ax
+	mov %ax, %ss
+	swapgs
+
+irq_skipgs1:
 	mov %rsp, %rdi
 	call irq_dispatch
 
+
 	POP_REGS
 	add $16, %rsp
+
+	cmpq $0x1b, 32(%rsp)
+	jne isr_skipgs2
+
+	swapgs
+
+irq_skipgs2:
+
 	iretq
 
 
@@ -363,12 +402,12 @@ irq_common_stub:
 
 syscall_gate:
 	swapgs
-	movq %rsp, %gs:20       # GS+20 = rsp2 in the current TSS block (user stack storage)
+	movq %rsp, %gs:94       # GS+20 = rsp2 in the current TSS block (user stack storage)
-	movq %gs:4, %rsp	# GS+4  = rsp0 in the current TSS block (per-thread kstack)
+	movq %gs:78, %rsp	# GS+4  = rsp0 in the current TSS block (per-thread kstack)
 
-	# start building a pf_cpu_context
+	# start building a ml_cpu_context
 	pushq $0x1b
-	pushq %gs:20
+	pushq %gs:94
 	push %r11
 	push $0x23
 	push %rcx
@@ -380,10 +419,6 @@ syscall_gate:
 
 	mov %rsp, %rdi
 
-	# switch back to user gs while in syscall_dispatch. Interrupts are enabled in syscall_dispatch,
-	# and if the task gets pre-empted, the incoming task will expect %gs to have its usermode value.
-	swapgs
-	
 	call syscall_dispatch
 
 	POP_REGS
@@ -394,8 +429,8 @@ syscall_gate:
 	pop %r11
 	add $16, %rsp
 
-	swapgs
+	movq %gs:94, %rsp       # GS+20 = rsp2 in the current TSS block
-	movq %gs:20, %rsp       # GS+20 = rsp2 in the current TSS block
+
 	swapgs
 
 	# back to usermode

arch/x86_64/panic.c

@@ -1,9 +1,12 @@
 #include <arch/irq.h>
-#include <mango/libc/stdio.h>
+#include <kernel/address-space.h>
-#include <mango/machine/cpu.h>
+#include <kernel/libc/stdio.h>
-#include <mango/machine/panic.h>
+#include <kernel/machine/cpu.h>
-#include <mango/printk.h>
+#include <kernel/machine/panic.h>
-#include <mango/vm.h>
+#include <kernel/printk.h>
+#include <kernel/sched.h>
+#include <kernel/task.h>
+#include <kernel/vm.h>
 
 #define R_CF  0
 #define R_PF  2
@@ -166,36 +169,66 @@ static void print_stack_item(uintptr_t addr)
 	printk("%s", buf);
 }
 
-static void print_stack_trace(uintptr_t ip, uintptr_t *bp)
+static bool read_stack_frame(
+	struct address_space *space,
+	uintptr_t bp,
+	struct stack_frame *out)
 {
-	struct stack_frame *stk = (struct stack_frame *)bp;
+	if (bp >= VM_PAGEMAP_BASE) {
+		*out = *(struct stack_frame *)bp;
+		return true;
+	}
+
+	if (!space) {
+		return false;
+	}
+
+	size_t tmp;
+	kern_status_t status
+		= address_space_read(space, bp, sizeof *out, out, &tmp);
+	return status == KERN_OK;
+}
+
+static void print_stack_trace(
+	struct address_space *space,
+	uintptr_t ip,
+	uintptr_t bp)
+{
+	struct stack_frame stk;
+	if (!read_stack_frame(space, bp, &stk)) {
+		return;
+	}
 	printk("call trace:");
 
 	print_stack_item(ip);
 
 	int max_frames = 10, current_frame = 0;
-	while (1) {
+	while (current_frame < max_frames) {
-		if (!vm_virt_to_phys(stk) || bp == NULL
+		uintptr_t addr = stk.rip;
-		    || current_frame > max_frames) {
-			break;
-		}
-
-		uintptr_t addr = stk->rip;
 		print_stack_item(addr);
 
-		stk = (struct stack_frame *)stk->rbp;
+		bp = stk.rbp;
+		if (!read_stack_frame(space, bp, &stk)) {
+			break;
+		}
 		current_frame++;
 	}
 }
 
 void ml_print_stack_trace(uintptr_t ip)
 {
-	uintptr_t *bp;
+	struct task *task = get_current_task();
+	struct address_space *space = task ? task->t_address_space : NULL;
+	uintptr_t bp;
 	asm volatile("mov %%rbp, %0" : "=r"(bp));
-	print_stack_trace(ip, bp);
+	print_stack_trace(space, ip, bp);
+	put_current_task(task);
 }
 
 void ml_print_stack_trace_irq(struct ml_cpu_context *ctx)
 {
-	print_stack_trace(ctx->rip, (uintptr_t *)ctx->rbp);
+	struct task *task = get_current_task();
+	struct address_space *space = task ? task->t_address_space : NULL;
+	print_stack_trace(space, ctx->rip, ctx->rbp);
+	put_current_task(task);
 }

arch/x86_64/pit.c

@@ -1,8 +1,8 @@
 #include <arch/irq.h>
 #include <arch/ports.h>
-#include <mango/clock.h>
+#include <kernel/clock.h>
-#include <mango/cpu.h>
+#include <kernel/cpu.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
 
 #define PIT_COUNTER0  0x40
 #define PIT_CMD       0x43

arch/x86_64/pmap.c

@@ -1,14 +1,15 @@
-#include <mango/compiler.h>
+#include <kernel/address-space.h>
-#include <mango/libc/stdio.h>
+#include <kernel/compiler.h>
-#include <mango/memblock.h>
+#include <kernel/libc/stdio.h>
-#include <mango/pmap.h>
+#include <kernel/memblock.h>
-#include <mango/printk.h>
+#include <kernel/pmap.h>
-#include <mango/sched.h>
+#include <kernel/printk.h>
-#include <mango/status.h>
+#include <kernel/sched.h>
-#include <mango/types.h>
+#include <kernel/task.h>
-#include <mango/vm-object.h>
+#include <kernel/types.h>
-#include <mango/vm-region.h>
+#include <kernel/vm-object.h>
-#include <mango/vm.h>
+#include <kernel/vm.h>
+#include <magenta/status.h>
 
 /* some helpful datasize constants */
 #define C_1GiB          0x40000000ULL
@@ -18,7 +19,7 @@
 #define PTR_TO_ENTRY(x) (((x) & ~VM_PAGE_MASK) | PTE_PRESENT | PTE_RW | PTE_USR)
 #define ENTRY_TO_PTR(x) ((x) & ~VM_PAGE_MASK)
 
-#define PFN(x)          ((x) >> VM_PAGE_SHIFT)
+#define PFN(x)          (((x) >> VM_PAGE_SHIFT) & 0xFFFFFFFFFF)
 
 static int can_use_gbpages = 0;
 static pmap_t kernel_pmap;
@@ -43,7 +44,7 @@ static pmap_t alloc_pmap(void)
 	return vm_virt_to_phys(p);
 }
 
-static pte_t make_pte(pfn_t pfn, enum vm_prot prot, enum page_size size)
+static pte_t make_pte(pfn_t pfn, vm_prot_t prot, enum page_size size)
 {
 	pte_t v = pfn;
 
@@ -99,7 +100,7 @@ static void delete_ptab(phys_addr_t pt)
 		return;
 	}
 
-	pt &= ~VM_PAGE_MASK;
+	pt = ENTRY_TO_PTR(pt);
 	if (!pt) {
 		/* physical address of 0x0, nothing to delete */
 		return;
@@ -116,7 +117,7 @@ static void delete_pdir(phys_addr_t pd)
 		return;
 	}
 
-	pd &= ~0x1FFFFFULL;
+	pd &= ENTRY_TO_PTR(pd);
 	if (!pd) {
 		/* physical address of 0x0, nothing to delete */
 		return;
@@ -135,11 +136,86 @@ static void delete_pdir(phys_addr_t pd)
 	kfree(pdir);
 }
 
+kern_status_t pmap_get(
+	pmap_t pmap,
+	virt_addr_t pv,
+	pfn_t *out_pfn,
+	vm_prot_t *out_prot)
+{
+	unsigned int pml4t_index = BAD_INDEX, pdpt_index = BAD_INDEX,
+		     pd_index = BAD_INDEX, pt_index = BAD_INDEX;
+
+	pml4t_index = (pv >> 39) & 0x1FF;
+	pdpt_index = (pv >> 30) & 0x1FF;
+	pd_index = (pv >> 21) & 0x1FF;
+	pt_index = (pv >> 12) & 0x1FF;
+
+	/* 1. get PML4T (mandatory) */
+	struct pml4t *pml4t = vm_phys_to_virt(ENTRY_TO_PTR(pmap));
+	if (!pml4t) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	/* 2. traverse PML4T, get PDPT (mandatory) */
+	struct pdpt *pdpt = NULL;
+	if (!pml4t->p_entries[pml4t_index]) {
+		return KERN_NO_ENTRY;
+	} else {
+		pdpt = vm_phys_to_virt(
+			ENTRY_TO_PTR(pml4t->p_entries[pml4t_index]));
+	}
+
+	/* 3. traverse PDPT, get PDIR (optional, 4K and 2M only) */
+	struct pdir *pdir = NULL;
+	if (!pdpt->p_entries[pdpt_index]
+	    || pdpt->p_pages[pdpt_index] & PTE_PAGESIZE) {
+		return KERN_NO_ENTRY;
+	} else {
+		pdir = vm_phys_to_virt(
+			ENTRY_TO_PTR(pdpt->p_entries[pdpt_index]));
+	}
+
+	/* 4. traverse PDIR, get PTAB (optional, 4K only) */
+	struct ptab *ptab = NULL;
+	if (!pdir->p_entries[pd_index]
+	    || pdir->p_pages[pd_index] & PTE_PAGESIZE) {
+		/* entry is null, or points to a hugepage */
+		return KERN_NO_ENTRY;
+	} else {
+		ptab = vm_phys_to_virt(ENTRY_TO_PTR(pdir->p_entries[pd_index]));
+	}
+
+	uint64_t pte = ptab->p_pages[pt_index];
+	if (out_pfn) {
+		*out_pfn = PFN(pte);
+	}
+
+	if (out_prot) {
+		if (pte & PTE_PRESENT) {
+			*out_prot |= VM_PROT_USER;
+		}
+
+		if (pte & PTE_RW) {
+			*out_prot |= (VM_PROT_READ | VM_PROT_WRITE);
+		}
+
+		if (pte & PTE_USR) {
+			*out_prot |= VM_PROT_USER;
+		}
+
+		if (!(pte & PTE_NX)) {
+			*out_prot |= VM_PROT_EXEC;
+		}
+	}
+
+	return KERN_OK;
+}
+
 static kern_status_t do_pmap_add(
 	pmap_t pmap,
 	virt_addr_t pv,
 	pfn_t pfn,
-	enum vm_prot prot,
+	vm_prot_t prot,
 	enum page_size size)
 {
 	unsigned int pml4t_index = BAD_INDEX, pdpt_index = BAD_INDEX,
@@ -240,6 +316,108 @@ static kern_status_t do_pmap_add(
 	return KERN_OK;
 }
 
+static kern_status_t do_pmap_remove(
+	pmap_t pmap,
+	virt_addr_t pv,
+	enum page_size size)
+{
+	if (pmap == PMAP_INVALID) {
+		return KERN_OK;
+	}
+
+	unsigned int pml4t_index = BAD_INDEX, pdpt_index = BAD_INDEX,
+		     pd_index = BAD_INDEX, pt_index = BAD_INDEX;
+
+	switch (size) {
+	case PS_4K:
+		pml4t_index = (pv >> 39) & 0x1FF;
+		pdpt_index = (pv >> 30) & 0x1FF;
+		pd_index = (pv >> 21) & 0x1FF;
+		pt_index = (pv >> 12) & 0x1FF;
+		break;
+	case PS_2M:
+		pml4t_index = (pv >> 39) & 0x1FF;
+		pdpt_index = (pv >> 30) & 0x1FF;
+		pd_index = (pv >> 21) & 0x1FF;
+		break;
+	case PS_1G:
+		if (!can_use_gbpages) {
+			return KERN_UNSUPPORTED;
+		}
+		pml4t_index = (pv >> 39) & 0x1FF;
+		pdpt_index = (pv >> 30) & 0x1FF;
+		break;
+	default:
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	/* 1. get PML4T (mandatory) */
+	struct pml4t *pml4t = vm_phys_to_virt(ENTRY_TO_PTR(pmap));
+	if (!pml4t) {
+		return KERN_OK;
+	}
+
+	/* 2. traverse PML4T, get PDPT (mandatory) */
+	struct pdpt *pdpt = NULL;
+	if (!pml4t->p_entries[pml4t_index]) {
+		return KERN_OK;
+	} else {
+		pdpt = vm_phys_to_virt(
+			ENTRY_TO_PTR(pml4t->p_entries[pml4t_index]));
+	}
+
+	/* if we're mapping a 1GiB page, we stop here */
+	if (size == PS_1G) {
+		if (pdpt->p_entries[pdpt_index] != 0) {
+			/* this slot points to a pdir, delete it.
+			   if this slot points to a hugepage, this does nothing
+			 */
+			delete_pdir(pdpt->p_entries[pdpt_index]);
+		}
+
+		pdpt->p_pages[pdpt_index] = 0;
+
+		return KERN_OK;
+	}
+
+	/* 3. traverse PDPT, get PDIR (optional, 4K and 2M only) */
+	struct pdir *pdir = NULL;
+	if (!pdpt->p_entries[pdpt_index]
+	    || pdpt->p_pages[pdpt_index] & PTE_PAGESIZE) {
+		/* entry is null, or points to a hugepage */
+		return KERN_OK;
+	} else {
+		pdir = vm_phys_to_virt(
+			ENTRY_TO_PTR(pdpt->p_entries[pdpt_index]));
+	}
+
+	/* if we're mapping a 2MiB page, we stop here */
+	if (size == PS_2M) {
+		if (pdir->p_entries[pd_index] != 0) {
+			/* this slot points to a ptab, delete it.
+			   if this slot points to a hugepage, this does nothing
+			 */
+			delete_ptab(pdir->p_entries[pd_index]);
+		}
+
+		pdir->p_pages[pd_index] = 0;
+		return KERN_OK;
+	}
+
+	/* 4. traverse PDIR, get PTAB (optional, 4K only) */
+	struct ptab *ptab = NULL;
+	if (!pdir->p_entries[pd_index]
+	    || pdir->p_pages[pd_index] & PTE_PAGESIZE) {
+		/* entry is null, or points to a hugepage */
+		return KERN_OK;
+	} else {
+		ptab = vm_phys_to_virt(ENTRY_TO_PTR(pdir->p_entries[pd_index]));
+	}
+
+	ptab->p_pages[pt_index] = 0;
+	return KERN_OK;
+}
+
 pmap_t get_kernel_pmap(void)
 {
 	return kernel_pmap;
@@ -252,6 +430,8 @@ void pmap_bootstrap(void)
 	       can_use_gbpages == 1 ? "en" : "dis");
 	enable_nx();
 	printk("pmap: NX protection enabled");
+	enable_wp();
+	printk("pmap: kernel-mode write protection enabled");
 
 	enum page_size hugepage = PS_2M;
 	if (can_use_gbpages) {
@@ -313,8 +493,46 @@ pmap_t pmap_create(void)
 	return pmap;
 }
 
+static void delete_pdpt(phys_addr_t pd)
+{
+	if (pd & PTE_PAGESIZE) {
+		/* this entry points to a hugepage, nothing to delete */
+		return;
+	}
+
+	pd &= ENTRY_TO_PTR(pd);
+	if (!pd) {
+		/* physical address of 0x0, nothing to delete */
+		return;
+	}
+
+	struct pdpt *pdpt = vm_phys_to_virt(ENTRY_TO_PTR(pd));
+	for (int i = 0; i < 512; i++) {
+		if (pdpt->p_pages[i] & PTE_PAGESIZE) {
+			/* this is a hugepage, there is nothing to delete */
+			continue;
+		}
+
+		if (!pdpt->p_entries[i]) {
+			continue;
+		}
+
+		delete_ptab(pdpt->p_entries[i]);
+	}
+
+	kfree(pdpt);
+}
+
 void pmap_destroy(pmap_t pmap)
 {
+	struct pml4t *pml4t = vm_phys_to_virt(ENTRY_TO_PTR(pmap));
+	for (unsigned int i = 0; i < 256; i++) {
+		if (pml4t->p_entries[i]) {
+			delete_pdpt(pml4t->p_entries[i]);
+		}
+	}
+
+	kfree(pml4t);
 }
 
 static void log_fault(virt_addr_t fault_addr, enum pmap_fault_flags flags)
@@ -357,21 +575,29 @@ kern_status_t pmap_handle_fault(
 {
 	// log_fault(fault_addr, flags);
 
-	if (flags & PMAP_FAULT_PRESENT) {
+	struct task *task = get_current_task();
+	if (!task) {
 		return KERN_FATAL_ERROR;
 	}
 
-	struct task *task = current_task();
+	struct address_space *space = task->t_address_space;
-	struct vm_region *space = task->t_address_space;
+	if (!space) {
+		return KERN_FATAL_ERROR;
+	}
 
-	return vm_region_demand_map(space, fault_addr, flags);
+	/* this must be called with `space` unlocked. */
+	kern_status_t status
+		= address_space_demand_map(space, fault_addr, flags);
+
+	put_current_task(task);
+	return status;
 }
 
 kern_status_t pmap_add(
 	pmap_t pmap,
 	virt_addr_t p,
 	pfn_t pfn,
-	enum vm_prot prot,
+	vm_prot_t prot,
 	enum pmap_flags flags)
 {
 	enum page_size ps = PS_4K;
@@ -387,18 +613,26 @@ kern_status_t pmap_add_block(
 	virt_addr_t p,
 	pfn_t pfn,
 	size_t len,
-	enum vm_prot prot,
+	vm_prot_t prot,
 	enum pmap_flags flags)
 {
 	return KERN_OK;
 }
 
-kern_status_t pmap_remove(pmap_t pmap, void *p)
+kern_status_t pmap_remove(pmap_t pmap, virt_addr_t p)
 {
+	return do_pmap_remove(pmap, p, PS_4K);
+}
+
+kern_status_t pmap_remove_range(pmap_t pmap, virt_addr_t p, size_t len)
+{
+	if (pmap == PMAP_INVALID) {
 		return KERN_OK;
 	}
 
-kern_status_t pmap_remove_range(pmap_t pmap, void *p, size_t len)
+	for (size_t i = p; i < p + len; i += VM_PAGE_SIZE) {
-{
+		pmap_remove(pmap, i);
+	}
+
 	return KERN_OK;
 }

arch/x86_64/pmap_ctrl.S

@@ -1,3 +1,11 @@
+	.global pmap_flush
+	.type pmap_flush, @function
+
+pmap_flush:
+	mov %cr3, %rax
+	mov %rax, %cr3
+	ret
+
 
 	.global pmap_switch
 	.type pmap_switch, @function
@@ -6,6 +14,7 @@ pmap_switch:
 	mov %rdi, %cr3
 	ret
 
+
 	.global gigabyte_pages
 	.type gigabyte_pages, @function
 
@@ -30,6 +39,7 @@ gigabyte_pages:
 	pop %rbp
 	ret
 
+
 	.global enable_nx
 	.type enable_nx, @function
 
@@ -40,3 +50,14 @@ enable_nx:
 	wrmsr
 
 	ret
+
+
+	.global enable_wp
+	.type enable_wp, @function
+
+enable_wp:
+	mov %cr0, %rax
+	or $0x10000, %rax
+	mov %rax, %cr0
+
+	ret

arch/x86_64/random.S

@@ -0,0 +1,43 @@
+.code64
+
+.global ml_hwrng_available
+.type ml_hwrng_available, @function
+
+ml_hwrng_available:
+	push %rbp
+	mov %rsp, %rbp
+	push %rbx
+	push %rdx
+
+	mov $1, %eax
+	mov $0, %ecx
+
+	cpuid
+
+	shr $30, %ecx
+	and $1, %ecx
+	mov %ecx, %eax
+
+	pop %rdx
+	pop %rbx
+	pop %rbp
+	ret
+
+.global ml_hwrng_generate
+.type ml_hwrng_generate, @function
+
+ml_hwrng_generate:
+	push %rbp
+	mov %rsp, %rbp
+
+	mov $100, %rcx
+.retry:
+	rdrand %rax
+	jc .done
+	loop .retry
+
+.fail:
+	mov $0, %rax
+.done:
+	pop %rbp
+	ret

arch/x86_64/serial.c

@@ -1,8 +1,8 @@
 #include <arch/irq.h>
 #include <arch/ports.h>
 #include <arch/serial.h>
-#include <mango/libc/stdio.h>
+#include <kernel/libc/stdio.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
 
 #define COM1 0x3F8
 #define COM2 0x2F8
@@ -28,6 +28,10 @@ void serial_send_byte(int device, char out)
 
 	outportb(device, out);
 
+	if (device == COM1) {
+		outportb(0xe9, out);
+	}
+
 	while (!transmit_empty(device)) {
 		_count++;
 	}
@@ -138,6 +142,7 @@ static struct console serialcon = {
 
 static int serial_irq1(void)
 {
+#if 0
 	if (serial_received(COM1)) {
 		unsigned char c = serial_recv_byte(COM1);
 		printk("serial: COM1 received %c", c);
@@ -147,6 +152,7 @@ static int serial_irq1(void)
 		unsigned char c = serial_recv_byte(COM3);
 		printk("serial: COM3 received %c", c);
 	}
+#endif
 
 	return 0;
 }

arch/x86_64/thread.c

@@ -1,5 +1,15 @@
-#include <mango/machine/cpu.h>
+#include <arch/msr.h>
-#include <mango/machine/thread.h>
+#include <kernel/machine/cpu.h>
+#include <kernel/machine/thread.h>
+#include <kernel/thread.h>
+
+#define MAX_REG_ARGS 6
+#define REG_ARG_0    rdi
+#define REG_ARG_1    rsi
+#define REG_ARG_2    rdx
+#define REG_ARG_3    rcx
+#define REG_ARG_4    r8
+#define REG_ARG_5    r9
 
 /* this is the context information restored by ml_thread_switch.
  * since ml_thread_switch only jumps to kernel-mode, IRETQ isn't used,
@@ -23,19 +33,125 @@ void ml_thread_prepare_kernel_context(uintptr_t ip, uintptr_t *sp)
 	ctx->rfl = 0x202;
 }
 
-extern void ml_thread_prepare_user_context(
+extern kern_status_t ml_thread_prepare_user_context(
 	virt_addr_t ip,
 	virt_addr_t user_sp,
-	virt_addr_t *kernel_sp)
+	virt_addr_t *kernel_sp,
+	const uintptr_t *args,
+	size_t nr_args)
 {
 	(*kernel_sp) -= sizeof(struct ml_cpu_context);
 
 	struct ml_cpu_context *ctx = (struct ml_cpu_context *)(*kernel_sp);
+	memset(ctx, 0x0, sizeof *ctx);
 	ctx->rip = ip;
 	ctx->rsp = user_sp;
-	ctx->ss = 0x23;
+	ctx->ss = 0x1b;
-	ctx->cs = 0x1B;
+	ctx->cs = 0x23;
 	ctx->rflags = 0x202;
 	ctx->rdi = 0;   // arg 0
 	ctx->rsi = 0;   // arg 1
+
+	for (size_t i = 0; i < nr_args; i++) {
+		switch (i) {
+		case 0:
+			ctx->REG_ARG_0 = args[i];
+			break;
+		case 1:
+			ctx->REG_ARG_1 = args[i];
+			break;
+		case 2:
+			ctx->REG_ARG_2 = args[i];
+			break;
+		case 3:
+			ctx->REG_ARG_3 = args[i];
+			break;
+		case 4:
+			ctx->REG_ARG_4 = args[i];
+			break;
+		case 5:
+			ctx->REG_ARG_5 = args[i];
+			break;
+		default:
+			return KERN_INVALID_ARGUMENT;
+		}
+	}
+
+	return KERN_OK;
+}
+
+kern_status_t ml_thread_clone_user_context(
+	const struct ml_cpu_context *src_regs,
+	const struct ml_thread *src_ml,
+	struct ml_thread *dest_ml,
+	uintptr_t return_value,
+	virt_addr_t *kernel_sp)
+{
+	(*kernel_sp) -= sizeof(struct ml_cpu_context);
+
+	struct ml_cpu_context *regs = (struct ml_cpu_context *)(*kernel_sp);
+	memcpy(regs, src_regs, sizeof *regs);
+
+	regs->rax = return_value;
+
+	dest_ml->tr_fsbase = src_ml->tr_fsbase;
+	dest_ml->tr_gsbase = src_ml->tr_gsbase;
+
+	return KERN_OK;
+}
+
+kern_status_t ml_thread_config_get(
+	struct thread *thread,
+	kern_config_key_t key,
+	void *out,
+	size_t max)
+{
+	return KERN_OK;
+}
+
+kern_status_t ml_thread_config_set(
+	struct thread *thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len)
+{
+	struct thread *self = get_current_thread();
+	kern_status_t status = KERN_OK;
+
+	switch (key) {
+	case THREAD_CFG_FSBASE:
+		if (len != sizeof(thread->tr_ml.tr_fsbase)) {
+			status = KERN_INVALID_ARGUMENT;
+			break;
+		}
+
+		thread->tr_ml.tr_fsbase = *(virt_addr_t *)ptr;
+		if (thread == self) {
+			wrmsr(MSR_FS_BASE, thread->tr_ml.tr_fsbase);
+		}
+
+		break;
+	case THREAD_CFG_GSBASE:
+		if (len != sizeof(thread->tr_ml.tr_gsbase)) {
+			status = KERN_INVALID_ARGUMENT;
+			break;
+		}
+
+		thread->tr_ml.tr_gsbase = *(virt_addr_t *)ptr;
+		if (thread == self) {
+			/* we're in the kernel right now, so the user and kernel
+			 * gs-base registers are swapped. when we return to
+			 * usermode, this value will be swapped back into
+			 * the user gs-base register */
+			wrmsr(MSR_KERNEL_GS_BASE, thread->tr_ml.tr_gsbase);
+		}
+
+		break;
+	default:
+		status = KERN_INVALID_ARGUMENT;
+		break;
+	}
+
+	put_current_thread(self);
+	return status;
 }

arch/x86_64/thread_switch.S

@@ -13,6 +13,22 @@ ml_thread_switch:
 	push %rax
 	push %rcx
 	push %rdx
+
+	// set fs-base
+	mov $0xC0000100, %rcx
+	movq THREAD_fsbase(%rsi), %rax
+	movq THREAD_fsbase(%rsi), %rdx
+	shr $32, %rdx
+	wrmsr
+
+	// set (kernel) gs-base (it will be swapped back into user-gs-base at
+	// the end of this function)
+	mov $0xC0000102, %rcx
+	movq THREAD_gsbase(%rsi), %rax
+	movq THREAD_gsbase(%rsi), %rdx
+	shr $32, %rdx
+	wrmsr
+
 	push %rbx
 	pushq $0
 	push %rbp
@@ -73,4 +89,5 @@ ml_thread_switch_user:
 	pop %rax
 
 	add $16, %rsp
+	swapgs
 	iretq

arch/x86_64/toolchain.cmake

@@ -1,5 +1,5 @@
 # the name of the target operating system
-set(CMAKE_SYSTEM_NAME Mango)
+set(CMAKE_SYSTEM_NAME Magenta)
 
 # which compilers to use for C and C++
 set(CMAKE_C_COMPILER   x86_64-elf-gcc)

arch/x86_64/tss.c

@@ -1,8 +1,6 @@
-#include "arch/msr.h"
-
 #include <arch/gdt.h>
 #include <arch/tss.h>
-#include <mango/libc/string.h>
+#include <kernel/libc/string.h>
 
 static void tss_flush(int index)
 {
@@ -22,9 +20,6 @@ void tss_init(struct tss *tss, struct tss_ptr *ptr)
 void tss_load(struct tss *tss)
 {
 	tss_flush(TSS_GDT_INDEX);
-
-	uintptr_t kernel_gs_base_reg = 0xC0000102;
-	wrmsr(kernel_gs_base_reg, (uintptr_t)tss);
 }
 
 virt_addr_t tss_get_kstack(struct tss *tss)

arch/x86_64/vga.c

@@ -1,9 +1,9 @@
 #include <arch/irq.h>
 #include <arch/ports.h>
 #include <arch/serial.h>
-#include <mango/libc/stdio.h>
+#include <kernel/libc/stdio.h>
-#include <mango/machine/vm.h>
+#include <kernel/machine/vm.h>
-#include <mango/printk.h>
+#include <kernel/printk.h>
 
 struct vga_console {
 	uint16_t *vga_framebuffer;

ds/bitmap.c

@@ -1,5 +1,5 @@
-#include <mango/bitmap.h>
+#include <kernel/bitmap.h>
-#include <mango/libc/string.h>
+#include <kernel/libc/string.h>
 
 void bitmap_zero(unsigned long *map, unsigned long nbits)
 {

ds/btree.c

@@ -57,7 +57,7 @@
    provide a comparator function.
 */ 
    
-#include <mango/btree.h>
+#include <kernel/btree.h>
 #include <stddef.h>
 
 #define MAX(a, b) ((a) > (b) ? (a) : (b))

ds/queue.c

@@ -1,4 +1,4 @@
-#include <mango/queue.h>
+#include <kernel/queue.h>
 
 size_t queue_length(struct queue *q)
 {

ds/ringbuffer.c

@@ -1,173 +0,0 @@
-#include <mango/ringbuffer.h>
-#include <mango/sched.h>
-
-size_t ringbuffer_unread(struct ringbuffer *ring_buffer)
-{
-	if (ring_buffer->r_read_ptr == ring_buffer->r_write_ptr) {
-		return 0;
-	}
-
-	if (ring_buffer->r_read_ptr > ring_buffer->r_write_ptr) {
-		return (ring_buffer->r_size - ring_buffer->r_read_ptr)
-			+ ring_buffer->r_write_ptr;
-	} else {
-		return (ring_buffer->r_write_ptr - ring_buffer->r_read_ptr);
-	}
-}
-
-size_t ringbuffer_avail(struct ringbuffer *ring_buffer)
-{
-	if (ring_buffer->r_read_ptr == ring_buffer->r_write_ptr) {
-		return ring_buffer->r_size - 1;
-	}
-
-	if (ring_buffer->r_read_ptr > ring_buffer->r_write_ptr) {
-		return ring_buffer->r_read_ptr - ring_buffer->r_write_ptr - 1;
-	} else {
-		return (ring_buffer->r_size - ring_buffer->r_write_ptr)
-			+ ring_buffer->r_read_ptr - 1;
-	}
-}
-
-static inline void increment_read(struct ringbuffer *ring_buffer)
-{
-	ring_buffer->r_read_ptr++;
-	if (ring_buffer->r_read_ptr == ring_buffer->r_size) {
-		ring_buffer->r_read_ptr = 0;
-	}
-}
-
-static inline void increment_write(struct ringbuffer *ring_buffer)
-{
-	ring_buffer->r_write_ptr++;
-	if (ring_buffer->r_write_ptr == ring_buffer->r_size) {
-		ring_buffer->r_write_ptr = 0;
-	}
-}
-
-size_t ringbuffer_read(struct ringbuffer *ring_buffer, size_t size, void *p, mango_flags_t flags)
-{
-	if (!ring_buffer) {
-		return 0;
-	}
-
-	unsigned char *buffer = p;
-	unsigned long lock_flags;
-	size_t collected = 0;
-
-	while (collected < size) {
-		spin_lock_irqsave(&ring_buffer->r_lock, &lock_flags);
-		while (ringbuffer_unread(ring_buffer) > 0 && collected < size) {
-			buffer[collected] = ring_buffer->r_buffer[ring_buffer->r_read_ptr];
-			increment_read(ring_buffer);
-			collected++;
-		}
-
-		wakeup_queue(&ring_buffer->r_wait_writers);
-
-		if (flags & S_NOBLOCK) {
-			spin_unlock_irqrestore(&ring_buffer->r_lock, lock_flags);
-			break;
-		}
-
-		struct wait_item waiter;
-		wait_item_init(&waiter, current_thread());
-		thread_wait_begin(&waiter, &ring_buffer->r_wait_readers);
-		spin_unlock_irqrestore(&ring_buffer->r_lock, lock_flags);
-
-		if (collected < size) {
-			schedule(SCHED_NORMAL);
-		}
-
-		thread_wait_end(&waiter, &ring_buffer->r_wait_readers);
-	}
-
-	wakeup_queue(&ring_buffer->r_wait_writers);
-	return collected;
-}
-
-size_t ringbuffer_write(struct ringbuffer *ring_buffer, size_t size, const void *p, mango_flags_t flags)
-{
-	if (!ring_buffer || !size) {
-		return 0;
-	}
-
-	const unsigned char *buffer = p;
-	unsigned long lock_flags;
-	size_t written = 0;
-
-	while (written < size) {
-		spin_lock_irqsave(&ring_buffer->r_lock, &lock_flags);
-
-		while (ringbuffer_avail(ring_buffer) > 0 && written < size) {
-			ring_buffer->r_buffer[ring_buffer->r_write_ptr] = buffer[written];
-			increment_write(ring_buffer);
-			written++;
-		}
-
-		wakeup_queue(&ring_buffer->r_wait_readers);
-
-		if (flags & S_NOBLOCK) {
-			spin_unlock_irqrestore(&ring_buffer->r_lock, lock_flags);
-			break;
-		}
-
-		struct wait_item waiter;
-		wait_item_init(&waiter, current_thread());
-		thread_wait_begin(&waiter, &ring_buffer->r_wait_writers);
-		spin_unlock_irqrestore(&ring_buffer->r_lock, lock_flags);
-
-		if (written < size) {
-			schedule(SCHED_NORMAL);
-		}
-
-		thread_wait_end(&waiter, &ring_buffer->r_wait_writers);
-	}
-
-	wakeup_queue(&ring_buffer->r_wait_readers);
-	return written;
-}
-
-struct ringbuffer *ringbuffer_create(size_t size)
-{
-	struct ringbuffer *out = kzalloc(sizeof(struct ringbuffer), VM_NORMAL);
-	if (!out) {
-		return NULL;
-	}
-
-	if (ringbuffer_init(out, size) != KERN_OK) {
-		kfree(out);
-		return NULL;
-	}
-
-	return out;
-}
-
-void ringbuffer_destroy(struct ringbuffer *ring_buffer)
-{
-	ringbuffer_deinit(ring_buffer);
-	kfree(ring_buffer);
-}
-
-kern_status_t ringbuffer_init(struct ringbuffer *buf, size_t size)
-{
-	buf->r_buffer = kmalloc(size, VM_NORMAL);
-	if (!buf->r_buffer) {
-		return KERN_NO_MEMORY;
-	}
-
-	buf->r_write_ptr = 0;
-	buf->r_read_ptr = 0;
-	buf->r_size = size;
-	buf->r_lock = SPIN_LOCK_INIT;
-
-	return KERN_OK;
-}
-
-kern_status_t ringbuffer_deinit(struct ringbuffer *buf)
-{
-	kfree(buf->r_buffer);
-	buf->r_buffer = NULL;
-
-	return KERN_OK;
-}

include/kernel/address-space.h

@@ -0,0 +1,181 @@
+#ifndef KERNEL_ADDRESS_SPACE_H_
+#define KERNEL_ADDRESS_SPACE_H_
+
+#include <kernel/object.h>
+#include <kernel/pmap.h>
+#include <kernel/vm.h>
+
+#define ADDRESS_SPACE_COPY_ALL ((size_t)-1)
+#define ADDRESS_SPACE_F_
+
+struct address_space;
+struct vm_object;
+
+struct vm_area {
+	/* the vm-object mapped into this area.
+	 * if this is NULL, the vm_area represents an area of reserved memory.
+	 * it cannot be accessed, and mapping operations with MAP_ADDRESS_ANY
+	 * will avoid the area, but fixed address mappings in this area
+	 * will succeed. */
+	struct vm_object *vma_object;
+	/* the address space that this vm_area is a part of */
+	struct address_space *vma_space;
+	/* used to link to vm_object->vo_mappings */
+	struct queue_entry vma_object_entry;
+	/* the memory control flags applied to this area */
+	vm_flags_t vma_flags;
+	/* the memory protection flags applied to this area */
+	vm_prot_t vma_prot;
+	/* offset in bytes to the start of the object data that was mapped */
+	off_t vma_object_offset;
+	/* used to link to address_space->s_mappings */
+	struct btree_node vma_node;
+	/* address of the first byte in this area */
+	virt_addr_t vma_base;
+	/* address of the last byte in this area */
+	virt_addr_t vma_limit;
+};
+
+struct address_space {
+	struct object s_base;
+
+	/* address of the first byte in this address space */
+	virt_addr_t s_base_address;
+	/* address of the last byte in this address space */
+	virt_addr_t s_limit_address;
+
+	/* btree of struct vm_area representing mapped vm-objects.
+	 * sibling entries cannot overlap each other. */
+	struct btree s_mappings;
+	/* btree of struct vm_area representing reserved regions of the
+	 * address space.
+	 * reserved regions will not be automatically allocated by the kernel.
+	 * sibling entries cannot overlap each other.
+	 * overlap between s_mappings and s_reserved IS allowed. */
+	struct btree s_reserved;
+
+	/* the corresponding physical address space */
+	pmap_t s_pmap;
+};
+
+extern kern_status_t address_space_type_init(void);
+extern struct address_space *address_space_cast(struct object *obj);
+
+/* create a new vm-region, optionally within a parent region.
+ * `offset` is the byte offset within the parent region where the new region
+ * should start.
+ * if no parent is specified, `offset` is the absolute virtual address of the
+ * start of the region.
+ * in both cases, `len` is the length of the new region in bytes. */
+extern kern_status_t address_space_create(
+	virt_addr_t base,
+	virt_addr_t limit,
+	struct address_space **out);
+
+/* map a vm-object into a vm-region.
+ * [region_offset,length] must fall within exactly one region, and cannot span
+ * multiple sibling regions.
+ * if [region_offset,length] falls within a child region, the map operation
+ * will be transparently redirected to the relevant region.
+ * `prot` must be allowed both by the region into which the mapping is being
+ * created AND the vm-object being mapped. */
+extern kern_status_t address_space_map(
+	struct address_space *space,
+	virt_addr_t map_address,
+	struct vm_object *object,
+	off_t object_offset,
+	size_t length,
+	vm_flags_t flags,
+	vm_prot_t prot,
+	virt_addr_t *out);
+extern kern_status_t address_space_unmap(
+	struct address_space *region,
+	virt_addr_t base,
+	size_t length);
+
+/* reserve an area of the address space. the kernel will not place any
+ * new mappings in this area unless explicitly told to (i.e. by not using
+ * MAP_ADDRESS_ANY). Use MAP_ADDRESS_ANY to have the kernel allocate a region
+ * of the address space for you */
+extern kern_status_t address_space_reserve(
+	struct address_space *space,
+	virt_addr_t base,
+	size_t length,
+	virt_addr_t *out);
+/* release a previously reserved area of the address space. */
+extern kern_status_t address_space_release(
+	struct address_space *space,
+	virt_addr_t base,
+	size_t length);
+
+/* duplicate all of the mappings in `src` within `dest. the duplication will use
+ * copy-on-write; page data will not be copied until it is written to. */
+extern kern_status_t address_space_duplicate(
+	struct address_space *dest,
+	struct address_space *src);
+
+extern bool address_space_validate_access(
+	struct address_space *region,
+	virt_addr_t base,
+	size_t len,
+	vm_prot_t prot);
+
+/* find the mapping corresponding to the given virtual address, and page-in the
+ * necessary vm_page to allow the memory access to succeed. if the relevant
+ * vm-object page hasn't been allocated yet, it will be allocated here.
+ * this function must be called with `region` UNLOCKED and interrupts ENABLED.
+ */
+extern kern_status_t address_space_demand_map(
+	struct address_space *region,
+	virt_addr_t addr,
+	enum pmap_fault_flags flags);
+
+/* read data from the user-space area of a vm-region into a kernel-mode buffer
+ */
+extern kern_status_t address_space_read(
+	struct address_space *src_region,
+	virt_addr_t src_ptr,
+	size_t count,
+	void *dest,
+	size_t *nr_read);
+
+/* write data to the user-space area of a vm-region from a kernel-mode buffer
+ */
+extern kern_status_t address_space_write(
+	struct address_space *dst_region,
+	virt_addr_t dst_ptr,
+	size_t count,
+	const void *src,
+	size_t *nr_written);
+
+extern kern_status_t address_space_memmove(
+	struct address_space *dest_space,
+	virt_addr_t dest_ptr,
+	struct address_space *src_space,
+	virt_addr_t src_ptr,
+	size_t count,
+	size_t *nr_moved);
+
+extern kern_status_t address_space_memmove_v(
+	struct address_space *dest_space,
+	size_t dest_offset,
+	const kern_iovec_t *dest_iov,
+	size_t nr_dest_iov,
+	struct address_space *src_space,
+	size_t src_offset,
+	const kern_iovec_t *src_iov,
+	size_t nr_src_iov,
+	size_t bytes_to_move,
+	size_t *nr_bytes_moved);
+
+extern kern_status_t address_space_translate(
+	struct address_space *space,
+	virt_addr_t in,
+	phys_addr_t *out,
+	unsigned long *irq_flags);
+
+void address_space_dump(struct address_space *region);
+
+DEFINE_OBJECT_LOCK_FUNCTION(address_space, s_base)
+
+#endif

include/kernel/arg.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_ARG_H_
+#ifndef KERNEL_ARG_H_
-#define MANGO_ARG_H_
+#define KERNEL_ARG_H_
 
+#include <magenta/types.h>
 #include <stdbool.h>
-#include <mango/status.h>
 
 #define CMDLINE_MAX 4096
 

include/kernel/atomic.h

@@ -0,0 +1,66 @@
+#ifndef KERNEL_ATOMIC_H_
+#define KERNEL_ATOMIC_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+
+typedef int32_t atomic_t;
+
+/* load and return the value pointed to by `v` */
+static inline atomic_t atomic_load(atomic_t *v)
+{
+	return __atomic_load_n(v, __ATOMIC_ACQUIRE);
+}
+/* store the value `v` to the pointer `dest` */
+static inline void atomic_store(atomic_t *dest, atomic_t v)
+{
+	__atomic_store_n(dest, v, __ATOMIC_ACQUIRE);
+}
+/* store the value `v` to the pointer `dest`, and return the value previously
+ * stored at `dest` */
+static inline atomic_t atomic_exchange(atomic_t *dest, atomic_t v)
+{
+	return __atomic_exchange_n(dest, v, __ATOMIC_ACQUIRE);
+}
+/* compare the contents of `ptr` to the contents of `expected`.
+ * if they match, store the value `desired` to the pointer `ptr` and return
+ * true. if the do NOT match, store the value `*ptr` to the pointer `desired`
+ * and return false.
+ */
+static inline bool atomic_compare_exchange(
+	atomic_t *ptr,
+	atomic_t *expected,
+	atomic_t desired)
+{
+	return __atomic_compare_exchange_n(
+		ptr,
+		expected,
+		desired,
+		false,
+		__ATOMIC_ACQUIRE,
+		__ATOMIC_ACQUIRE);
+}
+
+/* perform the operation *ptr += val, and return the result */
+static inline atomic_t atomic_add_fetch(atomic_t *ptr, atomic_t val)
+{
+	return __atomic_add_fetch(ptr, val, __ATOMIC_ACQUIRE);
+}
+/* perform the operation *ptr -= val, and return the result */
+static inline atomic_t atomic_sub_fetch(atomic_t *ptr, atomic_t val)
+{
+	return __atomic_sub_fetch(ptr, val, __ATOMIC_ACQUIRE);
+}
+
+/* perform the operation *ptr += val, and return the previous value of *ptr */
+static inline atomic_t atomic_fetch_add(atomic_t *ptr, atomic_t val)
+{
+	return __atomic_fetch_add(ptr, val, __ATOMIC_ACQUIRE);
+}
+/* perform the operation *ptr -= val, and return the previous value of *ptr */
+static inline atomic_t atomic_fetch_sub(atomic_t *ptr, atomic_t val)
+{
+	return __atomic_fetch_sub(ptr, val, __ATOMIC_ACQUIRE);
+}
+
+#endif

include/kernel/bitmap.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_BITMAP_H_
+#ifndef KERNEL_BITMAP_H_
-#define MANGO_BITMAP_H_
+#define KERNEL_BITMAP_H_
 
 #include <stdbool.h>
 

include/kernel/bsp.h

@@ -1,9 +1,9 @@
-#ifndef MANGO_BSP_H_
+#ifndef KERNEL_BSP_H_
-#define MANGO_BSP_H_
+#define KERNEL_BSP_H_
 
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/types.h>
+#include <kernel/types.h>
 #include <stddef.h>
 #include <stdint.h>
 

include/kernel/btree.h

@@ -20,26 +20,34 @@
     software without specific prior written permission.
  */
 
-#ifndef MANGO_BTREE_H_
+#ifndef KERNEL_BTREE_H_
-#define MANGO_BTREE_H_
+#define KERNEL_BTREE_H_
 
+#include <stdbool.h>
+#include <stddef.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-/* if your custom structure contains a struct btree_node (i.e. it can be part of a btree),
+/* if your custom structure contains a struct btree_node (i.e. it can be part of
-   you can use this macro to convert a struct btree_node* to a your_type*
+   a btree), you can use this macro to convert a struct btree_node* to a
+   your_type*
 
-   @param t the name of your custom type (something that can be passed to offsetof)
+   @param t the name of your custom type (something that can be passed to
-   @param m the name of the struct btree_node member variable within your custom type.
+   offsetof)
-   @param v the struct btree_node pointer that you wish to convert. if this is NULL, NULL will be returned.
+   @param m the name of the struct btree_node member variable within your custom
+   type.
+   @param v the struct btree_node pointer that you wish to convert. if this is
+   NULL, NULL will be returned.
 */
-#define BTREE_CONTAINER(t, m, v) ((void *)((v) ? (uintptr_t)(v) - (offsetof(t, m)) : 0))
+#define BTREE_CONTAINER(t, m, v)                                               \
+	((void *)((v) ? (uintptr_t)(v) - (offsetof(t, m)) : 0))
 
 /* defines a simple node insertion function.
-   this function assumes that your nodes have simple integer keys that can be compared with the usual operators.
+   this function assumes that your nodes have simple integer keys that can be
+   compared with the usual operators.
 
    EXAMPLE:
    if you have a tree node type like this:
@@ -49,45 +57,67 @@ extern "C" {
              struct btree_node base;
      }
 
-   You would use the following call to generate an insert function for a tree with this node type:
+   You would use the following call to generate an insert function for a tree
+   with this node type:
 
-     BTREE_DEFINE_SIMPLE_INSERT(struct my_tree_node, base, key, my_tree_node_insert);
+     BTREE_DEFINE_SIMPLE_INSERT(struct my_tree_node, base, key,
+   my_tree_node_insert);
 
    Which would emit a function defined like:
 
-     static void my_tree_node_insert(struct btree *tree, struct my_tree_node *node);
+     static void my_tree_node_insert(struct btree *tree, struct my_tree_node
+   *node);
 
-   @param node_type your custom tree node type. usually a structure that contains a struct btree_node member.
+   @param node_type your custom tree node type. usually a structure that
-   @param container_node_member the name of the struct btree_node member variable within your custom type.
+   contains a struct btree_node member.
-   @param container_key_member the name of the key member variable within your custom type.
+   @param container_node_member the name of the struct btree_node member
+   variable within your custom type.
+   @param container_key_member the name of the key member variable within your
+   custom type.
    @param function_name the name of the function to generate.
 */
-#define BTREE_DEFINE_SIMPLE_INSERT(node_type, container_node_member, container_key_member, function_name) \
+#define BTREE_DEFINE_SIMPLE_INSERT(                                            \
+	node_type,                                                             \
+	container_node_member,                                                 \
+	container_key_member,                                                  \
+	function_name)                                                         \
 	void function_name(struct btree *tree, node_type *node)                \
 	{                                                                      \
 		if (!tree->b_root) {                                           \
 			tree->b_root = &node->container_node_member;           \
-			btree_insert_fixup(tree, &node->container_node_member);                           \
+			btree_insert_fixup(                                    \
+				tree,                                          \
+				&node->container_node_member);                 \
 			return;                                                \
 		}                                                              \
                                                                                \
 		struct btree_node *cur = tree->b_root;                         \
 		while (1) {                                                    \
-			node_type *cur_node = BTREE_CONTAINER(node_type, container_node_member, cur);     \
+			node_type *cur_node = BTREE_CONTAINER(                 \
+				node_type,                                     \
+				container_node_member,                         \
+				cur);                                          \
 			struct btree_node *next = NULL;                        \
                                                                                \
-			if (node->container_key_member > cur_node->container_key_member) {                \
+			if (node->container_key_member                         \
+			    > cur_node->container_key_member) {                \
 				next = btree_right(cur);                       \
                                                                                \
 				if (!next) {                                   \
-					btree_put_right(cur, &node->container_node_member);               \
+					btree_put_right(                       \
+						cur,                           \
+						&node->container_node_member); \
 					break;                                 \
 				}                                              \
-			} else if (node->container_key_member < cur_node->container_key_member) {         \
+			} else if (                                            \
+				node->container_key_member                     \
+				< cur_node->container_key_member) {            \
 				next = btree_left(cur);                        \
                                                                                \
 				if (!next) {                                   \
-					btree_put_left(cur, &node->container_node_member);                \
+					btree_put_left(                        \
+						cur,                           \
+						&node->container_node_member); \
 					break;                                 \
 				}                                              \
 			} else {                                               \
@@ -101,8 +131,8 @@ extern "C" {
 	}
 
 /* defines a node insertion function.
-   this function should be used for trees with complex node keys that cannot be directly compared.
+   this function should be used for trees with complex node keys that cannot be
-   a comparator for your keys must be supplied.
+   directly compared. a comparator for your keys must be supplied.
 
    EXAMPLE:
    if you have a tree node type like this:
@@ -112,7 +142,8 @@ extern "C" {
              struct btree_node base;
      }
 
-   You would need to define a comparator function or macro with the following signature:
+   You would need to define a comparator function or macro with the following
+   signature:
 
      int my_comparator(struct my_tree_node *a, struct my_tree_node *b);
 
@@ -122,33 +153,49 @@ extern "C" {
      return  0   if a == b
      return  1   if a > b
 
-   You would use the following call to generate an insert function for a tree with this node type:
+   You would use the following call to generate an insert function for a tree
+   with this node type:
 
-     BTREE_DEFINE_INSERT(struct my_tree_node, base, key, my_tree_node_insert, my_comparator);
+     BTREE_DEFINE_INSERT(struct my_tree_node, base, key, my_tree_node_insert,
+   my_comparator);
 
    Which would emit a function defined like:
 
-     static void my_tree_node_insert(struct btree *tree, struct my_tree_node *node);
+     static void my_tree_node_insert(struct btree *tree, struct my_tree_node
+   *node);
 
-   @param node_type your custom tree node type. usually a structure that contains a struct btree_node member.
+   @param node_type your custom tree node type. usually a structure that
-   @param container_node_member the name of the struct btree_node member variable within your custom type.
+   contains a struct btree_node member.
-   @param container_key_member the name of the key member variable within your custom type.
+   @param container_node_member the name of the struct btree_node member
+   variable within your custom type.
+   @param container_key_member the name of the key member variable within your
+   custom type.
    @param function_name the name of the function to generate.
-   @param comparator the name of a comparator function or functional-macro that conforms to the
+   @param comparator the name of a comparator function or functional-macro that
-                     requirements listed above.
+   conforms to the requirements listed above.
 */
-#define BTREE_DEFINE_INSERT(node_type, container_node_member, container_key_member, function_name, comparator) \
+#define BTREE_DEFINE_INSERT(                                                   \
+	node_type,                                                             \
+	container_node_member,                                                 \
+	container_key_member,                                                  \
+	function_name,                                                         \
+	comparator)                                                            \
 	void function_name(struct btree *tree, node_type *node)                \
 	{                                                                      \
 		if (!tree->b_root) {                                           \
 			tree->b_root = &node->container_node_member;           \
-			btree_insert_fixup(tree, &node->container_node_member);                                \
+			btree_insert_fixup(                                    \
+				tree,                                          \
+				&node->container_node_member);                 \
 			return;                                                \
 		}                                                              \
                                                                                \
 		struct btree_node *cur = tree->b_root;                         \
 		while (1) {                                                    \
-			node_type *cur_node = BTREE_CONTAINER(node_type, container_node_member, cur);          \
+			node_type *cur_node = BTREE_CONTAINER(                 \
+				node_type,                                     \
+				container_node_member,                         \
+				cur);                                          \
 			struct btree_node *next = NULL;                        \
 			int cmp = comparator(node, cur_node);                  \
                                                                                \
@@ -156,14 +203,18 @@ extern "C" {
 				next = btree_right(cur);                       \
                                                                                \
 				if (!next) {                                   \
-					btree_put_right(cur, &node->container_node_member);                    \
+					btree_put_right(                       \
+						cur,                           \
+						&node->container_node_member); \
 					break;                                 \
 				}                                              \
 			} else if (cmp == -1) {                                \
 				next = btree_left(cur);                        \
                                                                                \
 				if (!next) {                                   \
-					btree_put_left(cur, &node->container_node_member);                     \
+					btree_put_left(                        \
+						cur,                           \
+						&node->container_node_member); \
 					break;                                 \
 				}                                              \
 			} else {                                               \
@@ -177,7 +228,8 @@ extern "C" {
 	}
 
 /* defines a simple tree search function.
-   this function assumes that your nodes have simple integer keys that can be compared with the usual operators.
+   this function assumes that your nodes have simple integer keys that can be
+   compared with the usual operators.
 
    EXAMPLE:
    if you have a tree node type like this:
@@ -187,27 +239,40 @@ extern "C" {
              struct btree_node base;
      }
 
-   You would use the following call to generate a search function for a tree with this node type:
+   You would use the following call to generate a search function for a tree
+   with this node type:
 
-     BTREE_DEFINE_SIMPLE_GET(struct my_tree_node, int, base, key, my_tree_node_get);
+     BTREE_DEFINE_SIMPLE_GET(struct my_tree_node, int, base, key,
+   my_tree_node_get);
 
    Which would emit a function defined like:
 
      static struct my_tree_node *my_tree_node_get(struct btree *tree, int key);
 
-   @param node_type your custom tree node type. usually a structure that contains a struct btree_node member.
+   @param node_type your custom tree node type. usually a structure that
-   @param key_type the type name of the key embedded in your custom tree node type. this type must be
+   contains a struct btree_node member.
-                   compatible with the builtin comparison operators.
+   @param key_type the type name of the key embedded in your custom tree node
-   @param container_node_member the name of the struct btree_node member variable within your custom type.
+   type. this type must be compatible with the builtin comparison operators.
-   @param container_key_member the name of the key member variable within your custom type.
+   @param container_node_member the name of the struct btree_node member
+   variable within your custom type.
+   @param container_key_member the name of the key member variable within your
+   custom type.
    @param function_name the name of the function to generate.
 */
-#define BTREE_DEFINE_SIMPLE_GET(node_type, key_type, container_node_member, container_key_member, function_name) \
+#define BTREE_DEFINE_SIMPLE_GET(                                               \
+	node_type,                                                             \
+	key_type,                                                              \
+	container_node_member,                                                 \
+	container_key_member,                                                  \
+	function_name)                                                         \
 	node_type *function_name(struct btree *tree, key_type key)             \
 	{                                                                      \
 		struct btree_node *cur = tree->b_root;                         \
 		while (cur) {                                                  \
-			node_type *cur_node = BTREE_CONTAINER(node_type, container_node_member, cur);            \
+			node_type *cur_node = BTREE_CONTAINER(                 \
+				node_type,                                     \
+				container_node_member,                         \
+				cur);                                          \
 			if (key > cur_node->container_key_member) {            \
 				cur = btree_right(cur);                        \
 			} else if (key < cur_node->container_key_member) {     \
@@ -241,15 +306,23 @@ extern "C" {
 
      btree_foreach (struct my_tree_node, node, &my_tree, base) { ... }
 
-   @param iter_type the type name of the iterator variable. this should be the tree's node type, and shouldn't be a pointer.
+   @param iter_type the type name of the iterator variable. this should be the
+   tree's node type, and shouldn't be a pointer.
    @param iter_name the name of the iterator variable.
    @param tree_name a pointer to the tree to traverse.
-   @param node_member the name of the struct btree_node member variable within the tree node type.
+   @param node_member the name of the struct btree_node member variable within
+   the tree node type.
 */
 #define btree_foreach(iter_type, iter_name, tree_name, node_member)            \
-	for (iter_type *iter_name = BTREE_CONTAINER(iter_type, node_member, btree_first(tree_name)); \
+	for (iter_type *iter_name = BTREE_CONTAINER(                           \
+		     iter_type,                                                \
+		     node_member,                                              \
+		     btree_first(tree_name));                                  \
 	     iter_name;                                                        \
-	     iter_name = BTREE_CONTAINER(iter_type, node_member, btree_next(&((iter_name)->node_member))))
+	     iter_name = BTREE_CONTAINER(                                      \
+		     iter_type,                                                \
+		     node_member,                                              \
+		     btree_next(&((iter_name)->node_member))))
 
 /* perform an reverse in-order traversal of a binary tree
 
@@ -272,35 +345,43 @@ extern "C" {
 
      btree_foreach_r (struct my_tree_node, node, &my_tree, base) { ... }
 
-   @param iter_type the type name of the iterator variable. this should be the tree's node type, and shouldn't be a pointer.
+   @param iter_type the type name of the iterator variable. this should be the
+   tree's node type, and shouldn't be a pointer.
    @param iter_name the name of the iterator variable.
    @param tree_name a pointer to the tree to traverse.
-   @param node_member the name of the struct btree_node member variable within the tree node type.
+   @param node_member the name of the struct btree_node member variable within
+   the tree node type.
 */
 #define btree_foreach_r(iter_type, iter_name, tree_name, node_member)          \
-	for (iter_type *iter_name = BTREE_CONTAINER(iter_type, node_member, btree_last(tree_name)); \
+	for (iter_type *iter_name                                              \
+	     = BTREE_CONTAINER(iter_type, node_member, btree_last(tree_name)); \
 	     iter_name;                                                        \
-	     iter_name = BTREE_CONTAINER(iter_type, node_member, btree_prev(&((iter_name)->node_member))))
+	     iter_name = BTREE_CONTAINER(                                      \
+		     iter_type,                                                \
+		     node_member,                                              \
+		     btree_prev(&((iter_name)->node_member))))
 
-/* binary tree nodes. this *cannot* be used directly. you need to define a custom node type
+/* binary tree nodes. this *cannot* be used directly. you need to define a
-   that contains a member variable of type struct btree_node.
+   custom node type that contains a member variable of type struct btree_node.
 
-   you would then use the supplied macros to define functions to manipulate your custom binary tree.
+   you would then use the supplied macros to define functions to manipulate your
+   custom binary tree.
 */
 struct btree_node {
 	struct btree_node *b_parent, *b_left, *b_right;
 	unsigned short b_height;
 };
 
-/* binary tree. unlike struct btree_node, you can define variables of type struct btree. */
+/* binary tree. unlike struct btree_node, you can define variables of type
+ * struct btree. */
 struct btree {
 	struct btree_node *b_root;
 };
 
 /* re-balance a binary tree after an insertion operation.
 
-   NOTE that, if you define an insertion function using BTREE_DEFINE_INSERT or similar,
+   NOTE that, if you define an insertion function using BTREE_DEFINE_INSERT or
-   this function will automatically called for you.
+   similar, this function will automatically called for you.
 
    @param tree the tree to re-balance.
    @param node the node that was just inserted into the tree.
@@ -316,29 +397,42 @@ extern void btree_delete(struct btree *tree, struct btree_node *node);
 
 /* get the first node in a binary tree.
 
-   this will be the node with the smallest key (i.e. the node that is furthest-left from the root)
+   this will be the node with the smallest key (i.e. the node that is
+   furthest-left from the root)
 */
 extern struct btree_node *btree_first(struct btree *tree);
 
 /* get the last node in a binary tree.
 
-   this will be the node with the largest key (i.e. the node that is furthest-right from the root)
+   this will be the node with the largest key (i.e. the node that is
+   furthest-right from the root)
 */
 extern struct btree_node *btree_last(struct btree *tree);
-/* for any binary tree node, this function returns the node with the next-largest key value */
+/* for any binary tree node, this function returns the node with the
+ * next-largest key value */
 extern struct btree_node *btree_next(struct btree_node *node);
-/* for any binary tree node, this function returns the node with the next-smallest key value */
+/* for any binary tree node, this function returns the node with the
+ * next-smallest key value */
 extern struct btree_node *btree_prev(struct btree_node *node);
 
+static inline bool btree_empty(const struct btree *tree)
+{
+	return tree->b_root == NULL;
+}
+
 /* sets `child` as the immediate left-child of `parent` */
-static inline void btree_put_left(struct btree_node *parent, struct btree_node *child)
+static inline void btree_put_left(
+	struct btree_node *parent,
+	struct btree_node *child)
 {
 	parent->b_left = child;
 	child->b_parent = parent;
 }
 
 /* sets `child` as the immediate right-child of `parent` */
-static inline void btree_put_right(struct btree_node *parent, struct btree_node *child)
+static inline void btree_put_right(
+	struct btree_node *parent,
+	struct btree_node *child)
 {
 	parent->b_right = child;
 	child->b_parent = parent;

include/kernel/channel.h

@@ -0,0 +1,55 @@
+#ifndef KERNEL_CHANNEL_H_
+#define KERNEL_CHANNEL_H_
+
+#include <kernel/object.h>
+#include <kernel/sched.h>
+
+struct msg;
+
+struct channel {
+	struct object c_base;
+	unsigned int c_id;
+	unsigned int c_msg_waiting;
+	struct btree c_msg;
+	struct btree_node c_node;
+};
+
+extern kern_status_t channel_type_init(void);
+extern struct channel *channel_cast(struct object *obj);
+
+extern struct channel *channel_create(void);
+
+extern kern_status_t channel_enqueue_msg(
+	struct channel *channel,
+	struct msg *msg);
+
+extern kern_status_t channel_recv_msg(
+	struct channel *channel,
+	kern_msg_t *out_msg,
+	unsigned long *irq_flags);
+extern kern_status_t channel_reply_msg(
+	struct channel *channel,
+	msgid_t id,
+	const kern_msg_t *reply,
+	unsigned long *irq_flags);
+
+extern kern_status_t channel_read_msg(
+	struct channel *channel,
+	msgid_t msg,
+	size_t offset,
+	struct address_space *dest_region,
+	const kern_iovec_t *dest_iov,
+	size_t dest_iov_count,
+	size_t *nr_read);
+extern kern_status_t channel_write_msg(
+	struct channel *channel,
+	msgid_t msg,
+	size_t offset,
+	struct address_space *src_region,
+	const kern_iovec_t *src_iov,
+	size_t src_iov_count,
+	size_t *nr_written);
+
+DEFINE_OBJECT_LOCK_FUNCTION(channel, c_base)
+
+#endif

include/kernel/clock.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_CLOCK_H_
+#ifndef KERNEL_CLOCK_H_
-#define MANGO_CLOCK_H_
+#define KERNEL_CLOCK_H_
 
 #include <stdint.h>
 

include/kernel/compiler.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_COMPILER_H_
+#ifndef KERNEL_COMPILER_H_
-#define MANGO_COMPILER_H_
+#define KERNEL_COMPILER_H_
 
 #ifdef __cplusplus
 template <typename T>

include/kernel/console.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_CONSOLE_H_
+#ifndef KERNEL_CONSOLE_H_
-#define MANGO_CONSOLE_H_
+#define KERNEL_CONSOLE_H_
 
 /* The console system
 
@@ -14,9 +14,10 @@
    representing a serial port may allow both sending AND receiving over the
    port.
 */
-#include <mango/queue.h>
+#include <kernel/locks.h>
-#include <mango/locks.h>
+#include <kernel/queue.h>
-#include <mango/status.h>
+#include <magenta/status.h>
+#include <magenta/types.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/cpu.h

@@ -1,10 +1,11 @@
-#ifndef MANGO_CPU_H_
+#ifndef KERNEL_CPU_H_
-#define MANGO_CPU_H_
+#define KERNEL_CPU_H_
 
-#include <mango/types.h>
+#include <kernel/machine/cpu.h>
-#include <mango/machine/cpu.h>
+#include <kernel/sched.h>
+#include <kernel/types.h>
+#include <kernel/work.h>
 #include <stdint.h>
-#include <mango/sched.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/equeue.h

@@ -0,0 +1,34 @@
+#ifndef KERNEL_EQUEUE_H_
+#define KERNEL_EQUEUE_H_
+
+#include <kernel/locks.h>
+#include <kernel/object.h>
+#include <kernel/sched.h>
+#include <magenta/types.h>
+
+#define EQUEUE_PACKET_MAX 100
+
+enum equeue_flags {
+	EQUEUE_WAIT,
+	EQUEUE_DISCARD,
+};
+
+struct equeue {
+	struct object eq_base;
+	unsigned short eq_read_ptr, eq_write_ptr;
+	equeue_packet_t eq_packets[EQUEUE_PACKET_MAX];
+	struct waitqueue eq_wq;
+};
+
+extern kern_status_t equeue_type_init(void);
+extern struct equeue *equeue_cast(struct object *obj);
+
+extern struct equeue *equeue_create(void);
+
+extern kern_status_t equeue_enqueue(
+	struct equeue *q,
+	const equeue_packet_t *pkt,
+	enum equeue_flags flags);
+extern kern_status_t equeue_dequeue(struct equeue *q, equeue_packet_t *out);
+
+#endif

include/kernel/futex.h

@@ -0,0 +1,34 @@
+#ifndef KERNEL_FUTEX_H_
+#define KERNEL_FUTEX_H_
+
+#include <kernel/btree.h>
+#include <kernel/wait.h>
+#include <magenta/types.h>
+
+struct task;
+struct address_space;
+
+typedef uintptr_t futex_key_t;
+
+struct futex {
+	struct btree_node f_node;
+
+	futex_key_t f_key;
+	struct waitqueue f_waiters;
+};
+
+extern kern_status_t futex_init(void);
+extern kern_status_t futex_get(
+	kern_futex_t *futex,
+	futex_key_t *out,
+	unsigned int flags);
+extern kern_status_t futex_wait(
+	futex_key_t futex,
+	kern_futex_t new_val,
+	unsigned int flags);
+extern kern_status_t futex_wake(
+	futex_key_t futex,
+	size_t nwaiters,
+	unsigned int flags);
+
+#endif

include/kernel/handle.h

@@ -1,8 +1,10 @@
-#ifndef MANGO_HANDLE_H_
+#ifndef KERNEL_HANDLE_H_
-#define MANGO_HANDLE_H_
+#define KERNEL_HANDLE_H_
 
-#include <mango/bitmap.h>
+#include <kernel/bitmap.h>
-#include <mango/status.h>
+#include <magenta/status.h>
+#include <magenta/types.h>
+#include <stddef.h>
 #include <stdint.h>
 
 /* subtract 32 bytes to account for the handle bitmap */
@@ -11,15 +13,16 @@
 
 typedef uint32_t kern_handle_t;
 
+typedef uintptr_t handle_flags_t;
+
+struct task;
 struct object;
+struct address_space;
+struct handle_list;
 
 struct handle {
-	union {
 	struct object *h_object;
-		uint64_t __x;
+	handle_flags_t h_flags;
-	};
-
-	uint64_t h_flags;
 };
 
 struct handle_table {
@@ -41,16 +44,30 @@ struct handle_table {
 
 extern struct handle_table *handle_table_create(void);
 extern void handle_table_destroy(struct handle_table *tab);
+extern kern_status_t handle_table_duplicate(
+	struct handle_table *src,
+	struct handle_table **dest);
 
 extern kern_status_t handle_table_alloc_handle(
 	struct handle_table *tab,
+	kern_handle_t value,
 	struct handle **out_slot,
 	kern_handle_t *out_handle);
-extern void handle_table_free_handle(
+extern kern_status_t handle_table_free_handle(
 	struct handle_table *tab,
 	kern_handle_t handle);
 extern struct handle *handle_table_get_handle(
 	struct handle_table *tab,
 	kern_handle_t handle);
 
+extern kern_status_t handle_table_transfer(
+	struct address_space *dst_region,
+	struct handle_table *dst,
+	kern_msg_handle_t *dst_handles,
+	size_t dst_handles_max,
+	struct address_space *src_region,
+	struct handle_table *src,
+	kern_msg_handle_t *src_handles,
+	size_t src_handles_count);
+
 #endif

include/kernel/init.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_INIT_H_
+#ifndef KERNEL_INIT_H_
-#define MANGO_INIT_H_
+#define KERNEL_INIT_H_
 
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
-#include <mango/machine/init.h>
+#include <kernel/machine/init.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/iovec.h

@@ -0,0 +1,33 @@
+#ifndef KERNEL_IOVEC_H_
+#define KERNEL_IOVEC_H_
+
+#include <magenta/types.h>
+#include <stddef.h>
+
+struct address_space;
+
+struct iovec_iterator {
+	/* if this is set, we are iterating over a list of iovecs stored in
+	 * userspace, and must go through this region to retrieve the data. */
+	struct address_space *it_region;
+	const kern_iovec_t *it_vecs;
+	size_t it_nr_vecs;
+	size_t it_vec_ptr;
+
+	virt_addr_t it_base;
+	size_t it_len;
+};
+
+extern void iovec_iterator_begin(
+	struct iovec_iterator *it,
+	const kern_iovec_t *vecs,
+	size_t nr_vecs);
+extern void iovec_iterator_begin_user(
+	struct iovec_iterator *it,
+	struct address_space *address_space,
+	const kern_iovec_t *vecs,
+	size_t nr_vecs);
+
+extern void iovec_iterator_seek(struct iovec_iterator *it, size_t nr_bytes);
+
+#endif

include/kernel/locks.h

@@ -0,0 +1,87 @@
+#ifndef KERNEL_LOCKS_H_
+#define KERNEL_LOCKS_H_
+
+#include <kernel/compiler.h>
+#include <kernel/machine/hwlock.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef __aligned(8) ml_hwlock_t spin_lock_t;
+
+#define SPIN_LOCK_INIT                ML_HWLOCK_INIT
+
+#define spin_lock(lck)                ml_hwlock_lock(lck);
+#define spin_unlock(lck)              ml_hwlock_unlock(lck);
+
+#define spin_lock_irq(lck)            ml_hwlock_lock_irq(lck);
+#define spin_unlock_irq(lck)          ml_hwlock_unlock_irq(lck);
+
+#define spin_lock_irqsave(lck, flags) ml_hwlock_lock_irqsave(lck, flags);
+#define spin_unlock_irqrestore(lck, flags)                                     \
+	ml_hwlock_unlock_irqrestore(lck, flags);
+
+static inline void spin_lock_pair(spin_lock_t *a, spin_lock_t *b)
+{
+	if (a == b) {
+		spin_lock(a);
+	} else if (a < b) {
+		spin_lock(a);
+		spin_lock(b);
+	} else {
+		spin_lock(b);
+		spin_lock(a);
+	}
+}
+
+static inline void spin_unlock_pair(spin_lock_t *a, spin_lock_t *b)
+{
+	if (a == b) {
+		spin_unlock(a);
+	} else if (a < b) {
+		spin_unlock(b);
+		spin_unlock(a);
+	} else {
+		spin_unlock(a);
+		spin_unlock(b);
+	}
+}
+
+static inline void spin_lock_pair_irqsave(
+	spin_lock_t *a,
+	spin_lock_t *b,
+	unsigned long *flags)
+{
+	if (a == b) {
+		spin_lock_irqsave(a, flags);
+	} else if (a < b) {
+		spin_lock_irqsave(a, flags);
+		spin_lock(b);
+	} else {
+		spin_lock_irqsave(b, flags);
+		spin_lock(a);
+	}
+}
+
+static inline void spin_unlock_pair_irqrestore(
+	spin_lock_t *a,
+	spin_lock_t *b,
+	unsigned long flags)
+{
+	if (a == b) {
+		spin_unlock_irqrestore(a, flags);
+	} else if (a < b) {
+		spin_unlock(b);
+		spin_unlock_irqrestore(a, flags);
+	} else {
+		spin_unlock(a);
+		spin_unlock_irqrestore(b, flags);
+	}
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/kernel/memblock.h

@@ -19,11 +19,11 @@
     contributors may be used to endorse or promote products derived from this
     software without specific prior written permission.
  */
-#ifndef MANGO_MEMBLOCK_H_
+#ifndef KERNEL_MEMBLOCK_H_
-#define MANGO_MEMBLOCK_H_
+#define KERNEL_MEMBLOCK_H_
 
+#include <kernel/types.h>
 #include <limits.h>
-#include <mango/types.h>
 #include <stddef.h>
 
 #ifdef __cplusplus
@@ -338,6 +338,8 @@ extern void __next_memory_region(
 	phys_addr_t start,
 	phys_addr_t end);
 
+extern void memblock_dump(void);
+
 #ifdef __cplusplus
 }
 #endif

include/kernel/msg.h

@@ -0,0 +1,58 @@
+#ifndef KERNEL_MSG_H_
+#define KERNEL_MSG_H_
+
+#include <kernel/btree.h>
+#include <kernel/locks.h>
+#include <magenta/status.h>
+#include <magenta/types.h>
+
+struct port;
+struct thread;
+
+enum kmsg_status {
+	KMSG_WAIT_RECEIVE,
+	KMSG_WAIT_REPLY,
+	KMSG_REPLY_SENT,
+	KMSG_ASYNC,
+};
+
+struct msg {
+	spin_lock_t msg_lock;
+	enum kmsg_status msg_status;
+	struct btree_node msg_node;
+	msgid_t msg_id;
+	union {
+		/* only valid for asynchronous messages (msg_status ==
+		 * KMSG_ASYNC) */
+		struct {
+			koid_t msg_sender_port_id;
+			tid_t msg_sender_thread_id;
+		};
+		/* only valid for synchronous messages (msg_status !=
+		 * KMSG_ASYNC) */
+		struct {
+			struct port *msg_sender_port;
+			struct thread *msg_sender_thread;
+		};
+	};
+	kern_status_t msg_result;
+	kern_msg_type_t msg_type;
+
+	union {
+		/* msg_type = KERN_MSG_TYPE_DATA */
+		struct {
+			kern_msg_t msg_req, msg_resp;
+		};
+
+		/* msg_type = KERN_MSG_TYPE_EVENT */
+		struct {
+			kern_msg_event_type_t msg_event;
+		};
+	};
+};
+
+extern void msg_init(void);
+extern struct msg *msg_alloc(void);
+extern void msg_free(struct msg *msg);
+
+#endif

include/kernel/object.h

@@ -1,10 +1,11 @@
-#ifndef MANGO_OBJECT_H_
+#ifndef KERNEL_OBJECT_H_
-#define MANGO_OBJECT_H_
+#define KERNEL_OBJECT_H_
 
-#include <mango/flags.h>
+#include <kernel/atomic.h>
-#include <mango/locks.h>
+#include <kernel/locks.h>
-#include <mango/status.h>
+#include <kernel/vm.h>
-#include <mango/vm.h>
+#include <kernel/wait.h>
+#include <magenta/status.h>
 #include <stddef.h>
 
 #ifdef __cplusplus
@@ -31,6 +32,32 @@ extern "C" {
 		unsigned long flags)                                           \
 	{                                                                      \
 		object_unlock_irqrestore(&p->base, flags);                     \
+	}                                                                      \
+	static inline void object_name##_lock_pair(                            \
+		struct object_name *a,                                         \
+		struct object_name *b)                                         \
+	{                                                                      \
+		object_lock_pair(&a->base, &b->base);                          \
+	}                                                                      \
+	static inline void object_name##_unlock_pair(                          \
+		struct object_name *a,                                         \
+		struct object_name *b)                                         \
+	{                                                                      \
+		object_unlock_pair(&a->base, &b->base);                        \
+	}                                                                      \
+	static inline void object_name##_lock_pair_irqsave(                    \
+		struct object_name *a,                                         \
+		struct object_name *b,                                         \
+		unsigned long *flags)                                          \
+	{                                                                      \
+		object_lock_pair_irqsave(&a->base, &b->base, flags);           \
+	}                                                                      \
+	static inline void object_name##_unlock_pair_irqrestore(               \
+		struct object_name *a,                                         \
+		struct object_name *b,                                         \
+		unsigned long flags)                                           \
+	{                                                                      \
+		object_unlock_pair_irqrestore(&a->base, &b->base, flags);      \
 	}
 
 #define OBJECT_MAGIC    0xBADDCAFE
@@ -38,7 +65,7 @@ extern "C" {
 #define OBJECT_PATH_MAX 256
 
 #define OBJECT_CAST(to_type, to_type_member, p)                                \
-	((to_type *)((uintptr_t)p) - offsetof(to_type, to_type_member))
+	((to_type *)(((uintptr_t)p) - offsetof(to_type, to_type_member)))
 #define OBJECT_C_CAST(c_type, c_type_member, obj_type, objp)                   \
 	OBJECT_IS_TYPE(objp, obj_type)                                         \
 	? OBJECT_CAST(c_type, c_type_member, (objp)) : NULL
@@ -67,11 +94,13 @@ struct object_type {
 
 struct object {
 	uint32_t ob_magic;
+	koid_t ob_id;
 	struct object_type *ob_type;
 	spin_lock_t ob_lock;
-	unsigned int ob_refcount;
+	uint32_t ob_signals;
-	unsigned int ob_handles;
+	atomic_t ob_refcount;
 	struct queue_entry ob_list;
+	struct waitqueue ob_wq;
 } __aligned(sizeof(long));
 
 extern kern_status_t object_bootstrap(void);
@@ -81,13 +110,30 @@ extern kern_status_t object_type_unregister(struct object_type *p);
 extern struct object *object_create(struct object_type *type);
 extern struct object *object_ref(struct object *obj);
 extern void object_unref(struct object *obj);
-extern void object_add_handle(struct object *obj);
-extern void object_remove_handle(struct object *obj);
 extern void object_lock(struct object *obj);
 extern void object_unlock(struct object *obj);
 extern void object_lock_irqsave(struct object *obj, unsigned long *flags);
 extern void object_unlock_irqrestore(struct object *obj, unsigned long flags);
 
+extern void object_lock_pair(struct object *a, struct object *b);
+extern void object_unlock_pair(struct object *a, struct object *b);
+
+extern void object_lock_pair_irqsave(
+	struct object *a,
+	struct object *b,
+	unsigned long *flags);
+extern void object_unlock_pair_irqrestore(
+	struct object *a,
+	struct object *b,
+	unsigned long flags);
+
+extern void object_assert_signal(struct object *obj, uint32_t signals);
+extern void object_clear_signal(struct object *obj, uint32_t signals);
+extern void object_wait_signal(
+	struct object *obj,
+	uint32_t signals,
+	unsigned long *irq_flags);
+
 #ifdef __cplusplus
 }
 #endif

include/kernel/panic.h

@@ -1,7 +1,7 @@
-#ifndef MANGO_PANIC_H_
+#ifndef KERNEL_PANIC_H_
-#define MANGO_PANIC_H_
+#define KERNEL_PANIC_H_
 
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
 
 struct ml_cpu_context;
 

include/kernel/percpu.h

@@ -1,9 +1,9 @@
-#ifndef MANGO_PERCPU_H_
+#ifndef KERNEL_PERCPU_H_
-#define MANGO_PERCPU_H_
+#define KERNEL_PERCPU_H_
 
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/compiler.h>
+#include <kernel/compiler.h>
-#include <mango/sched.h>
+#include <kernel/sched.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/pmap.h

@@ -1,15 +1,14 @@
-#ifndef MANGO_PMAP_H_
+#ifndef KERNEL_PMAP_H_
-#define MANGO_PMAP_H_
+#define KERNEL_PMAP_H_
 
 /* all the functions declared in this file are defined in arch/xyz/pmap.c */
 
-#include <mango/machine/pmap.h>
+#include <kernel/machine/pmap.h>
-#include <mango/status.h>
+#include <kernel/vm.h>
-#include <mango/vm.h>
+#include <magenta/status.h>
 #include <stddef.h>
 
 #define PMAP_INVALID ML_PMAP_INVALID
-#define PFN(x)       ((x) >> VM_PAGE_SHIFT)
 
 #ifdef __cplusplus
 extern "C" {
@@ -53,26 +52,35 @@ extern pmap_t pmap_create(void);
 extern void pmap_destroy(pmap_t pmap);
 extern void pmap_switch(pmap_t pmap);
 
+extern void pmap_flush(void);
+extern void pmap_flush_page(virt_addr_t p);
+extern void pmap_flush_range(virt_addr_t start, size_t length);
+
 extern kern_status_t pmap_handle_fault(
 	virt_addr_t fault_addr,
 	enum pmap_fault_flags flags);
 
+extern kern_status_t pmap_get(
+	pmap_t pmap,
+	virt_addr_t p,
+	pfn_t *out_pfn,
+	vm_prot_t *out_prot);
 extern kern_status_t pmap_add(
 	pmap_t pmap,
 	virt_addr_t p,
 	pfn_t pfn,
-	enum vm_prot prot,
+	vm_prot_t prot,
 	enum pmap_flags flags);
 extern kern_status_t pmap_add_block(
 	pmap_t pmap,
 	virt_addr_t p,
 	pfn_t pfn,
 	size_t len,
-	enum vm_prot prot,
+	vm_prot_t prot,
 	enum pmap_flags flags);
 
-extern kern_status_t pmap_remove(pmap_t pmap, void *p);
+extern kern_status_t pmap_remove(pmap_t pmap, virt_addr_t p);
-extern kern_status_t pmap_remove_range(pmap_t pmap, void *p, size_t len);
+extern kern_status_t pmap_remove_range(pmap_t pmap, virt_addr_t p, size_t len);
 
 #ifdef __cplusplus
 }

include/kernel/port.h

@@ -0,0 +1,35 @@
+#ifndef KERNEL_PORT_H_
+#define KERNEL_PORT_H_
+
+#include <kernel/object.h>
+#include <kernel/sched.h>
+
+enum port_status {
+	/* port is not connected */
+	PORT_OFFLINE = 0,
+	/* port is connected and ready to send messages */
+	PORT_READY,
+};
+
+struct port {
+	struct object p_base;
+	enum port_status p_status;
+	struct channel *p_remote;
+};
+
+extern kern_status_t port_type_init(void);
+extern struct port *port_cast(struct object *obj);
+
+extern struct port *port_create(void);
+
+extern kern_status_t port_connect(struct port *port, struct channel *remote);
+extern kern_status_t port_disconnect(struct port *port);
+extern kern_status_t port_send_msg(
+	struct port *port,
+	const kern_msg_t *msg,
+	kern_msg_t *out_response,
+	unsigned long *lock_flags);
+
+DEFINE_OBJECT_LOCK_FUNCTION(port, p_base)
+
+#endif

include/kernel/printk.h

@@ -1,7 +1,9 @@
-#ifndef MANGO_PRINTK_H_
+#ifndef KERNEL_PRINTK_H_
-#define MANGO_PRINTK_H_
+#define KERNEL_PRINTK_H_
 
-#include <mango/console.h>
+#include <kernel/console.h>
+
+#undef TRACE
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/queue.h

@@ -1,7 +1,7 @@
-#ifndef MANGO_QUEUE_H_
+#ifndef KERNEL_QUEUE_H_
-#define MANGO_QUEUE_H_
+#define KERNEL_QUEUE_H_
 
-#include <mango/libc/string.h>
+#include <kernel/libc/string.h>
 #include <stdbool.h>
 
 #ifdef __cplusplus

include/kernel/sched.h

@@ -0,0 +1,100 @@
+#ifndef KERNEL_SCHED_H_
+#define KERNEL_SCHED_H_
+
+#include <kernel/btree.h>
+#include <kernel/handle.h>
+#include <kernel/locks.h>
+#include <kernel/queue.h>
+#include <kernel/types.h>
+#include <magenta/status.h>
+
+#define PRIO_MAX 32
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct channel;
+struct runqueue;
+
+enum sched_priority {
+	PRIO_IDLE = 4,
+	PRIO_SUBNORMAL = 6,
+	PRIO_NORMAL = 10,
+	PRIO_SUPERNORMAL = 14,
+	PRIO_HIGH = 18,
+	PRIO_REALTIME = 24,
+};
+
+enum sched_mode {
+	/* used when calling from non-interrupt context.
+	   threads that aren't in state THREAD_READY are
+	   removed from the runqueue. */
+	SCHED_NORMAL = 0,
+	/* used when calling from interrupt context.
+	   threads that aren't in state THREAD_READY are
+	   still added to the runqueue. */
+	SCHED_IRQ = 1,
+};
+
+struct runqueue {
+	struct queue rq_queues[PRIO_MAX];
+	uint32_t rq_readybits;
+	spin_lock_t rq_lock;
+	unsigned int rq_nthreads;
+
+	struct thread *rq_cur, *rq_idle;
+};
+
+struct timer {
+	struct queue_entry t_entry;
+	struct cpu_data *t_cpu;
+	struct thread *t_owner;
+	unsigned long t_expiry;
+	void (*t_callback)(struct timer *);
+};
+
+extern kern_status_t sched_init(void);
+extern void schedule(enum sched_mode mode);
+extern void preempt_disable(void);
+extern void preempt_enable(void);
+
+extern void rq_init(struct runqueue *rq);
+extern struct thread *rq_dequeue(struct runqueue *rq);
+extern void rq_enqueue(struct runqueue *rq, struct thread *thr);
+static inline void rq_lock(struct runqueue *rq, unsigned long *flags)
+{
+	spin_lock_irqsave(&rq->rq_lock, flags);
+}
+static inline void rq_unlock(struct runqueue *rq, unsigned long flags)
+{
+	spin_unlock_irqrestore(&rq->rq_lock, flags);
+}
+extern void rq_remove_thread(struct runqueue *rq, struct thread *thr);
+extern struct runqueue *cpu_rq(unsigned int cpu);
+
+extern cycles_t default_quantum(void);
+
+extern bool need_resched(void);
+extern struct task *get_current_task(void);
+extern struct thread *get_current_thread(void);
+extern void put_current_task(struct task *task);
+extern void put_current_thread(struct thread *thread);
+
+extern struct runqueue *select_rq_for_thread(struct thread *thr);
+extern void schedule_thread_on_cpu(struct thread *thr);
+
+extern void start_charge_period(void);
+extern void end_charge_period(void);
+
+extern void add_timer(struct timer *timer);
+extern void remove_timer(struct timer *timer);
+extern unsigned long schedule_timeout(unsigned long clock_ticks);
+extern unsigned long milli_sleep(unsigned long ms);
+extern void sleep_forever(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/kernel/syscall.h

@@ -0,0 +1,259 @@
+#ifndef KERNEL_SYSCALL_H_
+#define KERNEL_SYSCALL_H_
+
+#include <kernel/address-space.h>
+#include <kernel/handle.h>
+#include <kernel/task.h>
+#include <kernel/vm.h>
+#include <magenta/status.h>
+#include <magenta/syscall.h>
+
+#define validate_access(task, ptr, len, flags)                                 \
+	__validate_access(task, (const void *)ptr, len, flags)
+#define validate_access_r(task, ptr, len)                                      \
+	validate_access(task, ptr, len, VM_PROT_READ | VM_PROT_USER)
+#define validate_access_w(task, ptr, len)                                      \
+	validate_access(task, ptr, len, VM_PROT_WRITE | VM_PROT_USER)
+#define validate_access_rw(task, ptr, len)                                     \
+	validate_access(                                                       \
+		task,                                                          \
+		ptr,                                                           \
+		len,                                                           \
+		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER)
+
+static inline bool __validate_access(
+	struct task *task,
+	const void *ptr,
+	size_t len,
+	vm_prot_t flags)
+{
+	unsigned long irq_flags;
+	address_space_lock_irqsave(task->t_address_space, &irq_flags);
+	bool result = address_space_validate_access(
+		task->t_address_space,
+		(virt_addr_t)ptr,
+		len,
+		flags | VM_PROT_USER);
+	address_space_unlock_irqrestore(task->t_address_space, irq_flags);
+	return result;
+}
+
+extern kern_status_t sys_task_exit(int status);
+extern kern_status_t sys_task_self(kern_handle_t *out);
+extern kern_status_t sys_task_create(
+	kern_handle_t parent_handle,
+	task_flags_t flags,
+	const char *name,
+	size_t name_len,
+	kern_handle_t *out_task,
+	kern_handle_t *out_address_space);
+extern kern_status_t sys_task_create_thread(
+	kern_handle_t task,
+	virt_addr_t ip,
+	virt_addr_t sp,
+	uintptr_t *args,
+	size_t nr_args,
+	kern_handle_t *out_thread);
+extern kern_status_t sys_task_get_address_space(
+	kern_handle_t task,
+	kern_handle_t *out);
+extern kern_status_t sys_task_config_get(
+	kern_handle_t task,
+	kern_config_key_t key,
+	void *ptr,
+	size_t len);
+extern kern_status_t sys_task_config_set(
+	kern_handle_t task,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+extern kern_status_t sys_task_duplicate(
+	kern_handle_t *out_task,
+	kern_handle_t *out_address_space);
+
+extern kern_status_t sys_thread_self(kern_handle_t *out);
+extern kern_status_t sys_thread_start(kern_handle_t thread);
+extern kern_status_t sys_thread_exit(void);
+extern kern_status_t sys_thread_config_get(
+	kern_handle_t thread,
+	kern_config_key_t key,
+	void *ptr,
+	size_t len);
+extern kern_status_t sys_thread_config_set(
+	kern_handle_t thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+
+extern kern_status_t sys_vm_object_create(
+	const char *name,
+	size_t name_len,
+	size_t data_len,
+	vm_prot_t prot,
+	kern_handle_t *out);
+extern kern_status_t sys_vm_object_read(
+	kern_handle_t object,
+	void *dst,
+	off_t offset,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t sys_vm_object_write(
+	kern_handle_t object,
+	const void *src,
+	off_t offset,
+	size_t count,
+	size_t *nr_written);
+extern kern_status_t sys_vm_object_copy(
+	kern_handle_t dst,
+	off_t dst_offset,
+	kern_handle_t src,
+	off_t src_offset,
+	size_t count,
+	size_t *nr_copied);
+
+extern kern_status_t sys_address_space_read(
+	kern_handle_t region,
+	void *dst,
+	virt_addr_t base,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t sys_address_space_write(
+	kern_handle_t region,
+	const void *src,
+	virt_addr_t base,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t sys_address_space_map(
+	kern_handle_t region,
+	virt_addr_t map_address,
+	kern_handle_t object,
+	off_t object_offset,
+	size_t length,
+	vm_flags_t flags,
+	vm_prot_t prot,
+	virt_addr_t *out_base_address);
+extern kern_status_t sys_address_space_unmap(
+	kern_handle_t region,
+	virt_addr_t base,
+	size_t length);
+extern kern_status_t sys_address_space_reserve(
+	kern_handle_t region,
+	virt_addr_t base,
+	size_t length,
+	virt_addr_t *out_base_address);
+extern kern_status_t sys_address_space_release(
+	kern_handle_t region,
+	virt_addr_t base,
+	size_t length);
+
+extern kern_status_t sys_kern_log(const char *s);
+extern kern_status_t sys_kern_handle_close(kern_handle_t handle);
+extern kern_status_t sys_kern_handle_transfer(
+	kern_handle_t src_task_handle,
+	kern_handle_t src_handle,
+	kern_handle_t dest_task_handle,
+	kern_handle_t dest_handle,
+	unsigned int mode,
+	kern_handle_t *out_handle);
+extern kern_status_t sys_kern_handle_control(
+	kern_handle_t task,
+	kern_handle_t handle,
+	uint32_t set_mask,
+	uint32_t clear_mask,
+	uint32_t *out_flags);
+extern kern_status_t sys_kern_config_get(
+	kern_config_key_t key,
+	void *ptr,
+	size_t len);
+extern kern_status_t sys_kern_config_set(
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+
+extern kern_status_t sys_channel_create(unsigned int id, kern_handle_t *out);
+extern kern_status_t sys_port_create(kern_handle_t *out);
+extern kern_status_t sys_port_connect(
+	kern_handle_t port,
+	tid_t task_id,
+	unsigned int channel_id);
+extern kern_status_t sys_port_disconnect(kern_handle_t port);
+
+extern kern_status_t sys_msg_send(
+	kern_handle_t port,
+	const kern_msg_t *msg,
+	kern_msg_t *out_reply);
+extern kern_status_t sys_msg_recv(kern_handle_t channel, kern_msg_t *out_msg);
+
+extern kern_status_t sys_msg_reply(
+	kern_handle_t channel,
+	msgid_t id,
+	const kern_msg_t *msg);
+extern kern_status_t sys_msg_read(
+	kern_handle_t channel_handle,
+	msgid_t id,
+	size_t offset,
+	const kern_iovec_t *iov,
+	size_t iov_count,
+	size_t *nr_read);
+extern kern_status_t sys_msg_write(
+	kern_handle_t channel,
+	msgid_t id,
+	size_t offset,
+	const kern_iovec_t *in,
+	size_t nr_in,
+	size_t *nr_written);
+
+extern kern_status_t sys_kern_object_wait(
+	kern_wait_item_t *items,
+	size_t nr_items);
+extern kern_status_t sys_kern_object_query(
+	kern_handle_t object,
+	kern_object_info_t *out);
+
+extern kern_status_t sys_vm_controller_create(kern_handle_t *out);
+extern kern_status_t sys_vm_controller_recv(
+	kern_handle_t ctrl,
+	equeue_packet_vm_request_t *out);
+extern kern_status_t sys_vm_controller_recv_async(
+	kern_handle_t ctrl,
+	kern_handle_t eq,
+	equeue_key_t key);
+extern kern_status_t sys_vm_controller_create_object(
+	kern_handle_t ctrl,
+	const char *name,
+	size_t name_len,
+	equeue_key_t key,
+	size_t data_len,
+	vm_prot_t prot,
+	kern_handle_t *out);
+extern kern_status_t sys_vm_controller_prepare_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	kern_handle_t *out_vmo);
+extern kern_status_t sys_vm_controller_finish_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key);
+extern kern_status_t sys_vm_controller_detach_object(
+	kern_handle_t ctrl,
+	kern_handle_t vmo);
+extern kern_status_t sys_vm_controller_supply_pages(
+	kern_handle_t ctrl,
+	kern_handle_t dst_vmo,
+	off_t dst_offset,
+	kern_handle_t src_vmo,
+	off_t src_offset,
+	size_t count);
+
+extern kern_status_t sys_futex_wait(
+	kern_futex_t *futex,
+	kern_futex_t new_val,
+	unsigned int flags);
+extern kern_status_t sys_futex_wake(
+	kern_futex_t *futex,
+	unsigned int nr_waiters,
+	unsigned int flags);
+
+extern virt_addr_t syscall_get_function(unsigned int sysid);
+
+#endif

include/kernel/task.h

@@ -0,0 +1,90 @@
+#ifndef KERNEL_TASK_H_
+#define KERNEL_TASK_H_
+
+#include <kernel/handle.h>
+#include <kernel/object.h>
+#include <kernel/pmap.h>
+
+#define TASK_NAME_MAX 64
+#define PID_MAX       99999
+
+struct channel;
+
+enum task_state {
+	TASK_RUNNING,
+	TASK_STOPPED,
+};
+
+struct task {
+	struct object t_base;
+
+	struct task *t_parent;
+	long t_id;
+	enum task_state t_state;
+	char t_name[TASK_NAME_MAX];
+
+	pmap_t t_pmap;
+	struct address_space *t_address_space;
+	spin_lock_t t_handles_lock;
+	struct handle_table *t_handles;
+	struct btree t_channels, t_futex;
+
+	struct btree_node t_tasklist;
+	struct queue_entry t_child_entry;
+
+	size_t t_next_thread_id;
+	struct queue t_threads;
+	struct queue t_children;
+};
+
+extern struct task *task_alloc(void);
+extern struct task *task_cast(struct object *obj);
+extern struct task *task_create(
+	struct task *parent,
+	task_flags_t flags,
+	const char *name,
+	size_t name_len);
+static inline struct task *task_ref(struct task *task)
+{
+	return OBJECT_CAST(struct task, t_base, object_ref(&task->t_base));
+}
+static inline void task_unref(struct task *task)
+{
+	object_unref(&task->t_base);
+}
+extern void task_exit(int status);
+extern kern_status_t task_add_child(struct task *parent, struct task *child);
+extern kern_status_t task_add_channel(
+	struct task *task,
+	struct channel *channel,
+	unsigned int id);
+extern struct channel *task_get_channel(struct task *task, unsigned int id);
+extern struct task *task_from_tid(tid_t id);
+extern kern_status_t task_open_handle(
+	struct task *task,
+	struct object *obj,
+	handle_flags_t flags,
+	kern_handle_t *out);
+extern kern_status_t task_resolve_handle(
+	struct task *task,
+	kern_handle_t handle,
+	struct object **out_obj,
+	handle_flags_t *out_flags);
+extern kern_status_t task_config_get(
+	struct task *task,
+	kern_config_key_t key,
+	void *out,
+	size_t max);
+extern kern_status_t task_config_set(
+	struct task *task,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+extern kern_status_t task_close_handle(struct task *task, kern_handle_t handle);
+extern struct thread *task_create_thread(struct task *parent);
+extern struct task *kernel_task(void);
+extern struct task *idle_task(void);
+
+DEFINE_OBJECT_LOCK_FUNCTION(task, t_base)
+
+#endif

include/kernel/thread.h

@@ -0,0 +1,91 @@
+#ifndef KERNEL_THREAD_H_
+#define KERNEL_THREAD_H_
+
+#include <kernel/machine/thread.h>
+#include <kernel/msg.h>
+#include <kernel/object.h>
+#include <kernel/vm-controller.h>
+
+#define THREAD_KSTACK_ORDER VM_PAGE_4K
+
+struct ml_cpu_context;
+
+enum thread_state {
+	THREAD_READY = 1,
+	THREAD_SLEEPING = 2,
+	THREAD_STOPPED = 3,
+};
+
+enum thread_flags {
+	/* this thread has exhausted its quantum and is due to be re-scheduled.
+	 */
+	THREAD_F_NEED_RESCHED = 0x01u,
+	/* this thread is currently scheduled (i.e. is present on a runqueue) */
+	THREAD_F_SCHEDULED = 0x04u,
+};
+
+struct thread {
+	struct object tr_base;
+
+	enum thread_state tr_state;
+	enum thread_flags tr_flags;
+	struct task *tr_parent;
+
+	unsigned int tr_id;
+	unsigned int tr_prio;
+
+	cycles_t tr_charge_period_start;
+	cycles_t tr_quantum_cycles, tr_quantum_target;
+	cycles_t tr_total_cycles;
+
+	virt_addr_t tr_ip, tr_sp, tr_bp;
+	virt_addr_t tr_cpu_user_sp, tr_cpu_kernel_sp;
+
+	/* only valid within an interrupt or syscall context */
+	struct ml_cpu_context *tr_irqctx;
+
+	struct ml_thread tr_ml;
+	struct runqueue *tr_rq;
+
+	struct queue_entry tr_parent_entry;
+	struct queue_entry tr_rqentry;
+
+	struct vm_page *tr_kstack;
+};
+
+extern struct thread *thread_alloc(void);
+extern struct thread *thread_cast(struct object *obj);
+extern kern_status_t thread_init_kernel(struct thread *thr, virt_addr_t ip);
+extern kern_status_t thread_init_user(
+	struct thread *thr,
+	virt_addr_t ip,
+	virt_addr_t sp,
+	const uintptr_t *args,
+	size_t nr_args);
+extern kern_status_t thread_init_user_clone(
+	struct thread *thr,
+	const struct thread *src,
+	uintptr_t return_value);
+extern int thread_priority(struct thread *thr);
+extern void thread_awaken(struct thread *thr);
+extern void thread_exit(void);
+extern void thread_join(struct thread *thread, unsigned long *irq_flags);
+extern void thread_kill(struct thread *thread);
+extern void idle(void);
+extern struct thread *create_kernel_thread(void (*fn)(void));
+extern struct thread *create_idle_thread(void);
+
+extern kern_status_t thread_config_get(
+	struct thread *thread,
+	kern_config_key_t key,
+	void *out,
+	size_t max);
+extern kern_status_t thread_config_set(
+	struct thread *thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+
+DEFINE_OBJECT_LOCK_FUNCTION(thread, tr_base)
+
+#endif

include/kernel/types.h

@@ -1,18 +1,14 @@
-#ifndef MANGO_TYPES_H_
+#ifndef KERNEL_TYPES_H_
-#define MANGO_TYPES_H_
+#define KERNEL_TYPES_H_
 
+#include <magenta/types.h>
 #include <stddef.h>
 #include <stdint.h>
 
 #define CYCLES_MAX UINT64_MAX
 
-typedef uintptr_t phys_addr_t;
-typedef uintptr_t virt_addr_t;
 typedef uint64_t cycles_t;
 typedef uint64_t sectors_t;
-typedef uint64_t off_t;
-
-typedef unsigned int umode_t;
 
 struct boot_module {
 	phys_addr_t mod_base;

include/kernel/util.h

@@ -1,6 +1,7 @@
-#ifndef MANGO_UTIL_H_
+#ifndef KERNEL_UTIL_H_
-#define MANGO_UTIL_H_
+#define KERNEL_UTIL_H_
 
+#include <magenta/types.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -60,6 +61,7 @@ extern uint64_t host_to_little_u64(uint64_t v);
 extern uint64_t big_to_host_u64(uint64_t v);
 extern uint64_t little_to_host_u64(uint64_t v);
 
+extern void init_random(uint64_t seed);
 extern bool fill_random(void *buffer, unsigned int size);
 
 #ifdef __cplusplus

include/kernel/vm-controller.h

@@ -0,0 +1,98 @@
+#ifndef KERNEL_VM_CONTROLLER_H_
+#define KERNEL_VM_CONTROLLER_H_
+
+#include <kernel/locks.h>
+#include <kernel/object.h>
+#include <magenta/types.h>
+
+struct thread;
+struct equeue;
+struct vm_object;
+
+enum vm_request_status {
+	VM_REQUEST_PENDING = 0,
+	VM_REQUEST_IN_PROGRESS,
+	VM_REQUEST_COMPLETE,
+	VM_REQUEST_ASYNC,
+};
+
+struct vm_controller {
+	struct object vc_base;
+	/* tree of pending vm requests */
+	struct btree vc_requests;
+	struct equeue *vc_eq;
+	equeue_key_t vc_eq_key;
+	/* the number of page requests queued with status VM_REQUEST_PENDING.
+	 * used to assert/clear VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED */
+	size_t vc_requests_waiting;
+};
+
+struct vm_request {
+	uint64_t req_id;
+	unsigned int req_type;
+	enum vm_request_status req_status;
+	kern_status_t req_result;
+	spin_lock_t req_lock;
+	struct vm_object *req_object;
+	struct thread *req_sender;
+	/* this node is added to vm-controller vc_requests list */
+	struct btree_node req_node;
+	/* these values are used for VM_REQUEST_READ and VM_REQUEST_DIRTY */
+	off_t req_offset;
+	size_t req_length;
+};
+
+extern kern_status_t vm_controller_type_init(void);
+extern struct vm_controller *vm_controller_cast(struct object *obj);
+
+extern struct vm_controller *vm_controller_create(void);
+
+extern kern_status_t vm_controller_recv(
+	struct vm_controller *ctrl,
+	equeue_packet_vm_request_t *out);
+extern kern_status_t vm_controller_recv_async(
+	struct vm_controller *ctrl,
+	struct equeue *eq,
+	equeue_key_t key);
+
+extern kern_status_t vm_controller_create_object(
+	struct vm_controller *ctrl,
+	const char *name,
+	size_t name_len,
+	equeue_key_t key,
+	size_t data_len,
+	vm_prot_t prot,
+	struct vm_object **out);
+extern kern_status_t vm_controller_prepare_attach(
+	struct vm_controller *ctrl,
+	uint64_t req_id,
+	struct vm_object **out_vmo);
+extern kern_status_t vm_controller_finish_attach(
+	struct vm_controller *ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key);
+extern kern_status_t vm_controller_detach_object(
+	struct vm_controller *ctrl,
+	struct vm_object *vmo);
+extern kern_status_t vm_controller_supply_pages(
+	struct vm_controller *ctrl,
+	struct vm_object *dst,
+	off_t dst_offset,
+	struct vm_object *src,
+	off_t src_offset,
+	size_t count);
+extern void vm_controller_fulfill_requests(
+	struct vm_controller *ctrl,
+	equeue_key_t object,
+	off_t offset,
+	size_t length,
+	kern_status_t result);
+
+extern kern_status_t vm_controller_send_request(
+	struct vm_controller *ctrl,
+	struct vm_request *req,
+	unsigned long *irq_flags);
+
+DEFINE_OBJECT_LOCK_FUNCTION(vm_controller, vc_base)
+
+#endif

include/kernel/vm-object.h

@@ -0,0 +1,150 @@
+#ifndef KERNEL_VM_OBJECT_H_
+#define KERNEL_VM_OBJECT_H_
+
+#include <kernel/locks.h>
+#include <kernel/object.h>
+
+#define VM_OBJECT_NAME_MAX 64
+
+struct vm_controller;
+
+enum vm_object_flags {
+	/* the memory behind this vm-object wasn't allocated by us, and
+	 * therefore shouldn't be freed by us */
+	VMO_IN_PLACE = 0x01u,
+	/* this vm-object is/was attached to a vm-controller */
+	VMO_CONTROLLER = 0x02u,
+	/* when a vm-object is attached to a controller, and the ref-count of
+	 * the object falls to one (i.e. the last reference is the handle to
+	 * the object held by the server that created it), the object will
+	 * be detached, allowing the server to close the last handle to the
+	 * object and dispose of it. */
+	VMO_AUTO_DETACH = 0x04u,
+	/* this vmo is a duplicate of a vmo that is attached to a vm-controller.
+	 * the duplicate vmo is scheduled to be attached to the same controller,
+	 * but this won't actually happen until the controller is needed to
+	 * fulfill a page request. once the duplicate vmo has been attached to
+	 * the controller, this flag will be cleared. */
+	VMO_LAZY_ATTACH = 0x08u,
+
+	/* these flags are for use with vm_object_get_page */
+	/**************************************************/
+
+	/* if the relevant page hasn't been allocated yet, it will be allocated
+	 * and returned. if this flag isn't specified, NULL will be returned. */
+	VMO_ALLOCATE_MISSING_PAGE = 0x0100u,
+	/* if the vm-object is attached to a vm-controller, and the relevant
+	 * page is uncommitted, send a request to the vm-controller to provide
+	 * the missing page. will result in the vm-object being unlocked and
+	 * the current thread sleeping until the request is fulfilled. the
+	 * vm-object will be re-locked before the function returns. */
+	VMO_REQUEST_MISSING_PAGE = 0x0200u,
+};
+
+struct vm_object {
+	struct object vo_base;
+
+	char vo_name[VM_OBJECT_NAME_MAX];
+	enum vm_object_flags vo_flags;
+
+	/* queue of struct vm_region_mapping */
+	struct queue vo_mappings;
+
+	struct vm_controller *vo_ctrl;
+	equeue_key_t vo_key;
+	struct btree_node vo_ctrl_node;
+
+	/* memory protection flags. mappings of this vm_object can only
+	 * use a subset of the flags set in this mask. */
+	vm_prot_t vo_prot;
+
+	/* btree of vm_pages that have been allocated to this vm_object.
+	 * vm_page->p_vmo_offset and the size of each page is the bst key. */
+	struct btree vo_pages;
+	/* total length of the vm_object in bytes. */
+	size_t vo_size;
+};
+
+extern kern_status_t vm_object_type_init(void);
+extern struct vm_object *vm_object_cast(struct object *obj);
+
+/* create a vm_object with the specified length in bytes and protection flags.
+ * the length will be automatically rounded up to the nearest vm_object page
+ * order size. the actual page frames themselves won't be allocated until
+ * they are mapped and accessed. */
+extern struct vm_object *vm_object_create(
+	const char *name,
+	size_t name_len,
+	size_t data_len,
+	vm_prot_t prot);
+
+/* create a vm_object that represents the specified range of physical memory.
+ * the length will be automatically rounded up to the nearest vm_object page
+ * order size.
+ * NOTE this function assumes that the physical memory has already been
+ * reserved, and is not in use by any other kernel component. */
+extern struct vm_object *vm_object_create_in_place(
+	const char *name,
+	size_t name_len,
+	phys_addr_t base,
+	size_t data_len,
+	vm_prot_t prot);
+
+/* create a copy-on-write duplicate of a vm-object */
+extern struct vm_object *vm_object_duplicate_cow(struct vm_object *vmo);
+/* attach a copy-on-write duplicate of a vm-object to the vm-controller that
+ * controlled the original vm-object */
+extern kern_status_t vm_object_attach_cow(
+	struct vm_object *vmo,
+	unsigned long *irq_flags);
+
+/* prefetch any missing pages in the specified range of a vm-object.
+ * any lazy-allocated pages will be allocated.
+ * any missing pages will be requested from the vm-controller, if one is
+ * attached. */
+extern kern_status_t vm_object_prefetch(
+	struct vm_object *vo,
+	off_t offset,
+	size_t length,
+	unsigned long *irq_flags);
+
+extern struct vm_page *vm_object_get_page(
+	struct vm_object *vo,
+	off_t offset,
+	enum vm_object_flags flags,
+	unsigned long *irq_flags);
+extern kern_status_t vm_object_put_page(
+	struct vm_object *vo,
+	off_t offset,
+	struct vm_page *pg);
+
+extern kern_status_t vm_object_read(
+	struct vm_object *vo,
+	void *dst,
+	off_t offset,
+	size_t count,
+	size_t *nr_read);
+extern kern_status_t vm_object_write(
+	struct vm_object *vo,
+	const void *dst,
+	off_t offset,
+	size_t count,
+	size_t *nr_written);
+extern kern_status_t vm_object_copy(
+	struct vm_object *dst,
+	off_t dst_offset,
+	struct vm_object *src,
+	off_t src_offset,
+	size_t count,
+	size_t *nr_copied);
+extern kern_status_t vm_object_transfer(
+	struct vm_object *dst,
+	off_t dst_offset,
+	struct vm_object *src,
+	off_t src_offset,
+	size_t count,
+	size_t *nr_moved);
+
+DEFINE_OBJECT_LOCK_FUNCTION(vm_object, vo_base)
+
+#endif

include/kernel/vm.h

@@ -1,21 +1,20 @@
-#ifndef MANGO_VM_H_
+#ifndef KERNEL_VM_H_
-#define MANGO_VM_H_
+#define KERNEL_VM_H_
 
-#include <mango/bitmap.h>
+#include <kernel/atomic.h>
-#include <mango/btree.h>
+#include <kernel/bitmap.h>
-#include <mango/locks.h>
+#include <kernel/btree.h>
-#include <mango/machine/vm.h>
+#include <kernel/locks.h>
-#include <mango/queue.h>
+#include <kernel/machine/vm.h>
-#include <mango/status.h>
+#include <kernel/queue.h>
-#include <mango/types.h>
+#include <kernel/types.h>
+#include <magenta/status.h>
 #include <stddef.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-struct bcache;
-
 /* maximum number of NUMA nodes */
 #define VM_MAX_NODES            64
 /* maximum number of memory zones per node */
@@ -25,11 +24,6 @@ struct bcache;
 /* maximum number of sparse memory sectors */
 #define VM_MAX_SECTORS          8192
 
-/* maximum number of disk sectors that can be stored in a single
-   page. AKA the number of bits in the sector bitmap.
-   used by the block cache */
-#define VM_MAX_SECTORS_PER_PAGE 32
-
 #define VM_CHECK_ALIGN(p, mask) ((((p) & (mask)) == (p)) ? 1 : 0)
 
 #define VM_CACHE_INITIALISED(c) ((c)->c_obj_count != 0)
@@ -47,15 +41,6 @@ enum vm_model {
 	VM_MODEL_SPARSE,
 };
 
-enum vm_prot {
-	VM_PROT_READ = 0x01u,
-	VM_PROT_WRITE = 0x02u,
-	VM_PROT_EXEC = 0x04u,
-	VM_PROT_USER = 0x08u,
-	VM_PROT_SVR = 0x10u,
-	VM_PROT_NOCACHE = 0x20u,
-};
-
 enum vm_flags {
 	VM_NORMAL = 0x00u,
 	VM_GET_DMA = 0x01u,
@@ -216,7 +201,6 @@ struct vm_page {
 	/* owner-specific pointer */
 	union {
 		struct vm_slab *p_slab;
-		struct bcache *p_bcache;
 		void *p_priv0;
 	};
 
@@ -227,8 +211,6 @@ struct vm_page {
 	       lists.
 	     - vm_object uses it to maintain a btree of allocated pages keyed
 	       by offset/size.
-	     - the block cache uses this to maintain a tree of pages keyed by
-	       block number.
 	*/
 	union {
 		struct queue_entry p_list;
@@ -240,15 +222,13 @@ struct vm_page {
 	};
 
 	union {
-		/* used by bcache when sector size is < page size. bitmap of
+		/* how many vm_areas reference this vm_page. if >0, the page is
-		 * present/missing sectors */
+		 * subject to copy-on-write. */
-		DECLARE_BITMAP(p_blockbits, VM_MAX_SECTORS_PER_PAGE);
+		atomic_t p_cow_ref;
 		uint32_t p_priv2;
 	};
 
 	union {
-		/* sector address, used by bcache */
-		sectors_t p_blockid;
 		/* offset of this page within the vm_object it is a part of */
 		off_t p_vmo_offset;
 

include/kernel/wait.h

@@ -0,0 +1,54 @@
+#ifndef KERNEL_WAIT_H_
+#define KERNEL_WAIT_H_
+
+#include <kernel/locks.h>
+#include <kernel/queue.h>
+
+#define wait_event(wq, cond)                                                   \
+	({                                                                     \
+		struct thread *self = current_thread();                        \
+		struct wait_item waiter;                                       \
+		wait_item_init(&waiter, self);                                 \
+		for (;;) {                                                     \
+			thread_wait_begin(&waiter, wq);                        \
+			if (cond) {                                            \
+				break;                                         \
+			}                                                      \
+			schedule(SCHED_NORMAL);                                \
+		}                                                              \
+		thread_wait_end(&waiter, wq);                                  \
+	})
+
+struct wait_item {
+	struct thread *w_thread;
+	struct queue_entry w_entry;
+};
+
+struct waitqueue {
+	struct queue wq_waiters;
+	spin_lock_t wq_lock;
+};
+
+extern void wait_item_init(struct wait_item *item, struct thread *thr);
+extern void thread_wait_begin(struct wait_item *waiter, struct waitqueue *q);
+extern void thread_wait_end(struct wait_item *waiter, struct waitqueue *q);
+extern void thread_wait_begin_nosleep(
+	struct wait_item *waiter,
+	struct waitqueue *q);
+extern void thread_wait_end_nosleep(
+	struct wait_item *waiter,
+	struct waitqueue *q);
+extern void wait_on_queue(struct waitqueue *q);
+extern void wakeup_queue(struct waitqueue *q);
+extern void wakeup_n(struct waitqueue *q, size_t n);
+extern void wakeup_one(struct waitqueue *q);
+static inline bool waitqueue_empty(struct waitqueue *wq)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&wq->wq_lock, &flags);
+	bool result = queue_empty(&wq->wq_waiters);
+	spin_unlock_irqrestore(&wq->wq_lock, flags);
+	return result;
+}
+
+#endif

include/kernel/work.h

@@ -0,0 +1,37 @@
+#ifndef KERNEL_WORK_H_
+#define KERNEL_WORK_H_
+
+#include <kernel/locks.h>
+#include <kernel/queue.h>
+#include <stddef.h>
+
+struct work_item;
+
+typedef void (*work_func_t)(struct work_item *);
+
+struct work_item {
+	void *w_data;
+	work_func_t w_func;
+	struct queue_entry w_head;
+};
+
+struct worker_pool {
+	struct thread **wp_workers;
+	size_t wp_nworkers;
+};
+
+struct workqueue {
+	spin_lock_t wq_lock;
+	struct queue wq_queue; /* list of struct work_item */
+};
+
+extern void work_item_init(work_func_t func, void *data, struct work_item *out);
+extern void workqueue_init(struct workqueue *wq);
+extern struct worker_pool *worker_pool_create(size_t nworkers);
+extern struct worker_pool *global_worker_pool(void);
+extern bool schedule_work_on(struct workqueue *wq, struct work_item *work);
+extern bool schedule_work(struct work_item *work);
+
+extern void wake_workers(struct workqueue *wq, struct worker_pool *pool);
+
+#endif

include/mango/fb.h

@@ -1,49 +0,0 @@
-#ifndef MANGO_FB_H_
-#define MANGO_FB_H_
-
-#include <stdint.h>
-
-enum framebuffer_flags {
-	FB_MODE_RGB = 0x01u,
-	FB_MODE_VGATEXT = 0x02u,
-	FB_MODE_PALETTE = 0x04u,
-};
-
-struct framebuffer_bitfield {
-	uint32_t b_offset;
-	uint16_t b_length;
-};
-
-struct framebuffer_varinfo {
-	enum framebuffer_flags fb_flags;
-
-	uint32_t fb_xres;
-	uint32_t fb_yres;
-	uint32_t fb_bpp;
-	uint32_t fb_stride;
-
-	union {
-		struct {
-			uint32_t fb_xcells;
-			uint32_t fb_ycells;
-		};
-
-		struct {
-			struct framebuffer_bitfield fb_red;
-			struct framebuffer_bitfield fb_green;
-			struct framebuffer_bitfield fb_blue;
-			struct framebuffer_bitfield fb_alpha;
-		};
-
-		struct {
-			uintptr_t fb_palette_addr;
-			uint16_t fb_palette_nr_colours;
-		};
-	};
-};
-
-struct framebuffer_fixedinfo {
-	uint64_t fb_baseptr;
-};
-
-#endif

include/mango/flags.h

@@ -1,11 +0,0 @@
-#ifndef MANGO_FLAGS_H_
-#define MANGO_FLAGS_H_
-
-#include <stdint.h>
-
-typedef enum {
-	S_NORMAL = 0x00u,
-	S_NOBLOCK = 0x01u,
-} mango_flags_t;
-
-#endif

include/mango/input.h

@@ -1,184 +0,0 @@
-#ifndef MANGO_INPUT_H_
-#define MANGO_INPUT_H_
-
-#include <stdint.h>
-#include <mango/queue.h>
-#include <mango/status.h>
-
-enum input_event_hook_flags {
-	INPUT_HOOK_SQUASH_EVENT = 0x01u,
-};
-
-struct device;
-
-enum input_event_type {
-	INPUT_TYPE_UNKNOWN = 0x00u,
-	INPUT_TYPE_KEY     = 0x01u,
-	INPUT_TYPE_MOTION  = 0x02u,
-};
-
-enum input_event_motion_type {
-	INPUT_MOTION_TYPE_MOUSE  = 0x01u,
-	INPUT_MOTION_TYPE_SCROLL = 0x02u,
-};
-
-enum input_keycode {
-	KEY_UNKNOWN                = 0x00u,
-	KEY_A                      = 0x01u,
-	KEY_B                      = 0x02u,
-	KEY_C                      = 0x03u,
-	KEY_D                      = 0x04u,
-	KEY_E                      = 0x05u,
-	KEY_F                      = 0x06u,
-	KEY_G                      = 0x07u,
-	KEY_H                      = 0x08u,
-	KEY_I                      = 0x09u,
-	KEY_J                      = 0x0Au,
-	KEY_K                      = 0x0Bu,
-	KEY_L                      = 0x0Cu,
-	KEY_M                      = 0x0Du,
-	KEY_N                      = 0x0Eu,
-	KEY_O                      = 0x0Fu,
-	KEY_P                      = 0x10u,
-	KEY_Q                      = 0x11u,
-	KEY_R                      = 0x12u,
-	KEY_S                      = 0x13u,
-	KEY_T                      = 0x14u,
-	KEY_U                      = 0x15u,
-	KEY_V                      = 0x16u,
-	KEY_W                      = 0x17u,
-	KEY_X                      = 0x18u,
-	KEY_Y                      = 0x19u,
-	KEY_Z                      = 0x1Au,
-	KEY_KEY_1                  = 0x1Bu,
-	KEY_KEY_2                  = 0x1Cu,
-	KEY_KEY_3                  = 0x1Du,
-	KEY_KEY_4                  = 0x1Eu,
-	KEY_KEY_5                  = 0x1Fu,
-	KEY_KEY_6                  = 0x20u,
-	KEY_KEY_7                  = 0x21u,
-	KEY_KEY_8                  = 0x22u,
-	KEY_KEY_9                  = 0x23u,
-	KEY_KEY_0                  = 0x24u,
-	KEY_ENTER                  = 0x25u,
-	KEY_ESCAPE                 = 0x26u,
-	KEY_BACKSPACE              = 0x27u,
-	KEY_TAB                    = 0x28u,
-	KEY_SPACE                  = 0x29u,
-	KEY_MINUS                  = 0x2Au,
-	KEY_EQUALS                 = 0x2Bu,
-	KEY_LEFT_BRACE             = 0x2Cu,
-	KEY_RIGHT_BRACE            = 0x2Du,
-	KEY_BACKSLASH              = 0x2Eu,
-	KEY_NON_US_HASH            = 0x2Fu,
-	KEY_SEMICOLON              = 0x30u,
-	KEY_APOSTROPHE             = 0x31u,
-	KEY_GRAVE_ACCENT           = 0x32u,
-	KEY_COMMA                  = 0x33u,
-	KEY_DOT                    = 0x34u,
-	KEY_SLASH                  = 0x35u,
-	KEY_CAPS_LOCK              = 0x36u,
-	KEY_F1                     = 0x37u,
-	KEY_F2                     = 0x38u,
-	KEY_F3                     = 0x39u,
-	KEY_F4                     = 0x3Au,
-	KEY_F5                     = 0x3Bu,
-	KEY_F6                     = 0x3Cu,
-	KEY_F7                     = 0x3Du,
-	KEY_F8                     = 0x3Eu,
-	KEY_F9                     = 0x3Fu,
-	KEY_F10                    = 0x40u,
-	KEY_F11                    = 0x41u,
-	KEY_F12                    = 0x42u,
-	KEY_PRINT_SCREEN           = 0x43u,
-	KEY_SCROLL_LOCK            = 0x44u,
-	KEY_PAUSE                  = 0x45u,
-	KEY_INSERT                 = 0x46u,
-	KEY_HOME                   = 0x47u,
-	KEY_PAGE_UP                = 0x48u,
-	KEY_DELETE                 = 0x49u,
-	KEY_END                    = 0x4Au,
-	KEY_PAGE_DOWN              = 0x4Bu,
-	KEY_RIGHT                  = 0x4Cu,
-	KEY_LEFT                   = 0x4Du,
-	KEY_DOWN                   = 0x4Eu,
-	KEY_UP                     = 0x4Fu,
-	KEY_NUM_LOCK               = 0x50u,
-	KEY_KEYPAD_SLASH           = 0x51u,
-	KEY_KEYPAD_ASTERISK        = 0x52u,
-	KEY_KEYPAD_MINUS           = 0x53u,
-	KEY_KEYPAD_PLUS            = 0x54u,
-	KEY_KEYPAD_ENTER           = 0x55u,
-	KEY_KEYPAD_1               = 0x56u,
-	KEY_KEYPAD_2               = 0x57u,
-	KEY_KEYPAD_3               = 0x58u,
-	KEY_KEYPAD_4               = 0x59u,
-	KEY_KEYPAD_5               = 0x5Au,
-	KEY_KEYPAD_6               = 0x5Bu,
-	KEY_KEYPAD_7               = 0x5Cu,
-	KEY_KEYPAD_8               = 0x5Du,
-	KEY_KEYPAD_9               = 0x5Eu,
-	KEY_KEYPAD_0               = 0x5Fu,
-	KEY_KEYPAD_DOT             = 0x60u,
-	KEY_NON_US_BACKSLASH       = 0x61u,
-	KEY_KEYPAD_EQUALS          = 0x62u,
-	KEY_MENU                   = 0x63u,
-	KEY_LEFT_CTRL              = 0x64u,
-	KEY_LEFT_SHIFT             = 0x65u,
-	KEY_LEFT_ALT               = 0x66u,
-	KEY_LEFT_META              = 0x67u,
-	KEY_RIGHT_CTRL             = 0x68u,
-	KEY_RIGHT_SHIFT            = 0x69u,
-	KEY_RIGHT_ALT              = 0x6Au,
-	KEY_RIGHT_META             = 0x6Bu,
-	KEY_MEDIA_MUTE             = 0x6Cu,
-	KEY_MEDIA_VOLUME_INCREMENT = 0x6Du,
-	KEY_MEDIA_VOLUME_DECREMENT = 0x6Eu,
-};
-
-enum input_key_state {
-	INPUT_KEYSTATE_DOWN = 0x00u,
-	INPUT_KEYSTATE_UP   = 0x01u,
-};
-
-enum input_button {
-	INPUT_BUTTON_MOUSE_LEFT    = 0x00u,
-	INPUT_BUTTON_MOUSE_MIDDLE  = 0x01u,
-	INPUT_BUTTON_MOUSE_RIGHT   = 0x02u,
-	INPUT_BUTTON_MOUSE_BACK    = 0x03u,
-	INPUT_BUTTON_MOUSE_FORWARD = 0x04u,
-};
-
-enum input_button_state {
-	INPUT_BUTTON_DOWN = 0x00u,
-	INPUT_BUTTON_UP   = 0x01u,
-};
-
-struct input_event {
-	enum input_event_type ev_type;
-	union {
-		struct {
-			enum input_event_motion_type type;
-			int16_t movement_x;
-			int16_t movement_y;
-		} ev_motion;
-
-		struct {
-			enum input_button button;
-			enum input_button_state state;
-		} ev_button;
-
-		struct {
-			enum input_keycode key;
-			enum input_key_state state;
-		} ev_key;
-	};
-};
-
-struct input_event_hook {
-	void(*hook_callback)(struct device *, struct input_event *, enum input_event_hook_flags *, void *);
-	void *hook_arg;
-	struct queue_entry hook_head;
-};
-
-#endif

include/mango/locks.h

@@ -1,25 +0,0 @@
-#ifndef MANGO_LOCKS_H_
-#define MANGO_LOCKS_H_
-
-#include <mango/compiler.h>
-#include <mango/machine/hwlock.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef __aligned(8) ml_hwlock_t spin_lock_t;
-
-#define SPIN_LOCK_INIT ML_HWLOCK_INIT
-
-#define spin_lock(lck) ml_hwlock_lock(lck);
-#define spin_unlock(lck) ml_hwlock_unlock(lck);
-
-#define spin_lock_irqsave(lck, flags) ml_hwlock_lock_irqsave(lck, flags);
-#define spin_unlock_irqrestore(lck, flags) ml_hwlock_unlock_irqrestore(lck, flags);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

include/mango/ringbuffer.h

@@ -1,34 +0,0 @@
-#ifndef MANGO_RINGBUFFER_H_
-#define MANGO_RINGBUFFER_H_
-
-#include <mango/locks.h>
-#include <mango/sched.h>
-
-struct ringbuffer {
-	unsigned char *r_buffer;
-	size_t r_write_ptr;
-	size_t r_read_ptr;
-	size_t r_size;
-	spin_lock_t r_lock;
-	struct waitqueue r_wait_readers;
-	struct waitqueue r_wait_writers;
-};
-
-extern struct ringbuffer *ringbuffer_create(size_t size);
-extern void ringbuffer_destroy(struct ringbuffer *buf);
-
-extern kern_status_t ringbuffer_init(struct ringbuffer *buf, size_t size);
-extern kern_status_t ringbuffer_deinit(struct ringbuffer *buf);
-
-extern size_t ringbuffer_unread(struct ringbuffer *buf);
-extern size_t ringbuffer_avail(struct ringbuffer *buf);
-extern size_t ringbuffer_read(struct ringbuffer *buf, size_t size, void *buffer, mango_flags_t flags);
-extern size_t ringbuffer_write(struct ringbuffer *buf, size_t size, const void *buffer, mango_flags_t flags);
-
-/* TODO */
-//extern size_t ringbuffer_peek(struct ringbuffer *buf, size_t at, size_t size, void *buffer);
-//extern size_t ringbuffer_skip(struct ringbuffer *buf, size_t count);
-
-extern int ringbuffer_write_would_block(struct ringbuffer *buf);
-
-#endif

include/mango/sched.h

@@ -1,250 +0,0 @@
-#ifndef MANGO_SCHED_H_
-#define MANGO_SCHED_H_
-
-#include <mango/btree.h>
-#include <mango/locks.h>
-#include <mango/object.h>
-#include <mango/pmap.h>
-#include <mango/queue.h>
-#include <mango/status.h>
-
-#define TASK_NAME_MAX       64
-#define PRIO_MAX            32
-#define PID_MAX             99999
-#define THREAD_KSTACK_ORDER VM_PAGE_4K
-#define THREAD_MAX          65536
-
-#define wait_event(wq, cond)                                                   \
-	({                                                                     \
-		struct thread *self = current_thread();                        \
-		struct wait_item waiter;                                       \
-		wait_item_init(&waiter, self);                                 \
-		for (;;) {                                                     \
-			thread_wait_begin(&waiter, wq);                        \
-			if (cond) {                                            \
-				break;                                         \
-			}                                                      \
-			schedule(SCHED_NORMAL);                                \
-		}                                                              \
-		thread_wait_end(&waiter, wq);                                  \
-	})
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct runqueue;
-struct work_item;
-
-enum task_state {
-	TASK_RUNNING,
-	TASK_STOPPED,
-};
-
-enum thread_state {
-	THREAD_READY = 1,
-	THREAD_SLEEPING = 2,
-	THREAD_STOPPED = 3,
-};
-
-enum thread_flags {
-	THREAD_F_NEED_RESCHED = 0x01u,
-	THREAD_F_NO_PREEMPT = 0x02u,
-};
-
-enum sched_priority {
-	PRIO_IDLE = 4,
-	PRIO_SUBNORMAL = 6,
-	PRIO_NORMAL = 10,
-	PRIO_SUPERNORMAL = 14,
-	PRIO_HIGH = 18,
-	PRIO_REALTIME = 24,
-};
-
-enum sched_mode {
-	/* used when calling from non-interrupt context.
-	   threads that aren't in state THREAD_READY are
-	   removed from the runqueue. */
-	SCHED_NORMAL = 0,
-	/* used when calling from interrupt context.
-	   threads that aren't in state THREAD_READY are
-	   still added to the runqueue. */
-	SCHED_IRQ = 1,
-};
-
-struct task {
-	struct object t_base;
-
-	struct task *t_parent;
-	long t_id;
-	enum task_state t_state;
-	char t_name[TASK_NAME_MAX];
-
-	pmap_t t_pmap;
-	struct vm_region *t_address_space;
-	struct handle_table *t_handles;
-
-	struct btree_node t_tasklist;
-	struct queue_entry t_child_entry;
-
-	size_t t_next_thread_id;
-	struct queue t_threads;
-	struct queue t_children;
-};
-
-struct thread {
-	struct object thr_base;
-
-	enum thread_state tr_state;
-	enum thread_flags tr_flags;
-	struct task *tr_parent;
-
-	unsigned int tr_id;
-	unsigned int tr_prio;
-
-	cycles_t tr_charge_period_start;
-	cycles_t tr_quantum_cycles, tr_quantum_target;
-	cycles_t tr_total_cycles;
-
-	virt_addr_t tr_ip, tr_sp, tr_bp;
-	virt_addr_t tr_cpu_user_sp, tr_cpu_kernel_sp;
-
-	struct runqueue *tr_rq;
-
-	struct queue_entry tr_parent_entry;
-	struct queue_entry tr_rqentry;
-
-	struct vm_page *tr_kstack;
-	struct vm_object *tr_ustack;
-};
-
-struct runqueue {
-	struct queue rq_queues[PRIO_MAX];
-	uint32_t rq_readybits;
-	spin_lock_t rq_lock;
-	unsigned int rq_nthreads;
-
-	struct thread *rq_cur, *rq_idle;
-};
-
-struct timer {
-	struct queue_entry t_entry;
-	struct cpu_data *t_cpu;
-	struct thread *t_owner;
-	unsigned long t_expiry;
-	void (*t_callback)(struct timer *);
-};
-
-struct wait_item {
-	struct thread *w_thread;
-	struct queue_entry w_entry;
-};
-
-struct waitqueue {
-	struct queue wq_waiters;
-	spin_lock_t wq_lock;
-};
-
-typedef void (*work_func_t)(struct work_item *);
-
-struct work_item {
-	void *w_data;
-	work_func_t w_func;
-	struct queue_entry w_head;
-};
-
-struct worker_pool {
-	struct thread **wp_workers;
-	size_t wp_nworkers;
-};
-
-struct workqueue {
-	spin_lock_t wq_lock;
-	struct queue wq_queue; /* list of struct work_item */
-};
-
-extern kern_status_t sched_init(void);
-extern void schedule(enum sched_mode mode);
-extern void preempt_disable(void);
-extern void preempt_enable(void);
-
-extern void rq_init(struct runqueue *rq);
-extern struct thread *rq_dequeue(struct runqueue *rq);
-extern void rq_enqueue(struct runqueue *rq, struct thread *thr);
-static inline void rq_lock(struct runqueue *rq, unsigned long *flags)
-{
-	spin_lock_irqsave(&rq->rq_lock, flags);
-}
-static inline void rq_unlock(struct runqueue *rq, unsigned long flags)
-{
-	spin_unlock_irqrestore(&rq->rq_lock, flags);
-}
-extern void rq_remove_thread(struct runqueue *rq, struct thread *thr);
-extern struct runqueue *cpu_rq(unsigned int cpu);
-
-extern struct task *task_alloc(void);
-extern struct task *task_create(struct task *parent, const char *name);
-static inline struct task *task_ref(struct task *task)
-{
-	return OBJECT_CAST(struct task, t_base, object_ref(&task->t_base));
-}
-static inline void task_unref(struct task *task)
-{
-	object_unref(&task->t_base);
-}
-extern struct task *task_from_pid(unsigned int pid);
-extern struct thread *task_create_thread(struct task *parent);
-extern struct task *kernel_task(void);
-extern struct task *idle_task(void);
-extern cycles_t default_quantum(void);
-
-extern bool need_resched(void);
-extern struct task *current_task(void);
-extern struct thread *current_thread(void);
-
-extern struct runqueue *select_rq_for_thread(struct thread *thr);
-extern void schedule_thread_on_cpu(struct thread *thr);
-
-extern void start_charge_period(void);
-extern void end_charge_period(void);
-
-DEFINE_OBJECT_LOCK_FUNCTION(task, t_base)
-
-extern struct thread *thread_alloc(void);
-extern kern_status_t thread_init_kernel(struct thread *thr, virt_addr_t ip);
-extern kern_status_t thread_init_user(
-	struct thread *thr,
-	virt_addr_t ip,
-	virt_addr_t sp);
-extern int thread_priority(struct thread *thr);
-extern void idle(void);
-extern struct thread *create_kernel_thread(void (*fn)(void));
-extern struct thread *create_idle_thread(void);
-
-extern void add_timer(struct timer *timer);
-extern void remove_timer(struct timer *timer);
-extern unsigned long schedule_timeout(unsigned long clock_ticks);
-extern unsigned long milli_sleep(unsigned long ms);
-extern void sleep_forever(void);
-
-extern void wait_item_init(struct wait_item *item, struct thread *thr);
-extern void thread_wait_begin(struct wait_item *waiter, struct waitqueue *q);
-extern void thread_wait_end(struct wait_item *waiter, struct waitqueue *q);
-extern void wait_on_queue(struct waitqueue *q);
-extern void wakeup_queue(struct waitqueue *q);
-extern void wakeup_one(struct waitqueue *q);
-
-extern void work_item_init(work_func_t func, void *data, struct work_item *out);
-extern void workqueue_init(struct workqueue *wq);
-extern struct worker_pool *worker_pool_create(size_t nworkers);
-extern struct worker_pool *global_worker_pool(void);
-extern bool schedule_work_on(struct workqueue *wq, struct work_item *work);
-extern bool schedule_work(struct work_item *work);
-
-extern void wake_workers(struct workqueue *wq, struct worker_pool *pool);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

include/mango/syscall.h

@@ -1,20 +0,0 @@
-#ifndef MANGO_SYSCALL_H_
-#define MANGO_SYSCALL_H_
-
-#include <mango/handle.h>
-#include <mango/status.h>
-#include <mango/vm.h>
-
-#define SYS_EXIT             1
-#define SYS_VM_OBJECT_CREATE 2
-
-extern kern_status_t sys_exit(int status);
-extern kern_status_t sys_vm_object_create(
-	const char *name,
-	size_t len,
-	enum vm_prot prot,
-	kern_handle_t *out_handle);
-
-extern virt_addr_t syscall_get_function(unsigned int sysid);
-
-#endif

include/mango/test.h

@@ -1,14 +0,0 @@
-#ifndef MANGO_TEST_H_
-#define MANGO_TEST_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern int run_all_tests(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

include/mango/vm-object.h

@@ -1,68 +0,0 @@
-#ifndef MANGO_VM_OBJECT_H_
-#define MANGO_VM_OBJECT_H_
-
-#include <mango/locks.h>
-#include <mango/object.h>
-
-#define VM_OBJECT_NAME_MAX 64
-
-enum vm_object_flags {
-	/* the memory behind this vm-object wasn't allocated by us, and
-	 * therefore shouldn't be freed by us */
-	VMO_IN_PLACE = 0x01u,
-};
-
-struct vm_object {
-	struct object vo_base;
-
-	char vo_name[VM_OBJECT_NAME_MAX];
-	enum vm_object_flags vo_flags;
-
-	/* queue of struct vm_region_mapping */
-	struct queue vo_mappings;
-
-	/* memory protection flags. mappings of this vm_object can only use
-	 * a subset of the flags set in this mask. */
-	enum vm_prot vo_prot;
-
-	/* btree of vm_pages that have been allocated to this vm_object.
-	 * vm_page->p_vmo_offset and the size of each page is the bst key. */
-	struct btree vo_pages;
-	/* total length of the vm_object in bytes. */
-	size_t vo_size;
-};
-
-extern kern_status_t vm_object_type_init(void);
-
-/* create a vm_object with the specified length in bytes and protection flags.
- * the length will be automatically rounded up to the nearest vm_object page
- * order size. the actual page frames themselves won't be allocated until
- * they are mapped and accessed. */
-extern struct vm_object *vm_object_create(
-	const char *name,
-	size_t len,
-	enum vm_prot prot);
-
-/* create a vm_object that represents the specified range of physical memory.
- * the length will be automatically rounded up to the nearest vm_object page
- * order size.
- * NOTE this function assumes that the physical memory has already been
- * reserved, and is not in use by any other kernel component. */
-extern struct vm_object *vm_object_create_in_place(
-	const char *name,
-	phys_addr_t base,
-	size_t len,
-	enum vm_prot prot);
-
-extern struct vm_page *vm_object_get_page(
-	const struct vm_object *vo,
-	off_t offset);
-
-extern struct vm_page *vm_object_alloc_page(
-	struct vm_object *vo,
-	off_t offset,
-	enum vm_page_order size);
-
-DEFINE_OBJECT_LOCK_FUNCTION(vm_object, vo_base)
-
-#endif

include/mango/vm-region.h

@@ -1,115 +0,0 @@
-#ifndef MANGO_VM_REGION_H_
-#define MANGO_VM_REGION_H_
-
-#include <mango/object.h>
-#include <mango/pmap.h>
-#include <mango/vm.h>
-
-#define VM_REGION_NAME_MAX   64
-
-#define VM_REGION_ANY_OFFSET ((off_t) - 1)
-
-struct vm_region;
-struct vm_object;
-
-enum vm_region_entry_type {
-	VM_REGION_ENTRY_NONE = 0,
-	VM_REGION_ENTRY_REGION,
-	VM_REGION_ENTRY_MAPPING,
-};
-
-struct vm_region_entry {
-	struct btree_node e_node;
-	struct vm_region_entry *e_parent;
-	enum vm_region_entry_type e_type;
-	/* offset in bytes of this entry within its immediate parent. */
-	off_t e_offset;
-	/* size of the entry in bytes */
-	size_t e_size;
-};
-
-struct vm_region_mapping {
-	struct vm_region_entry m_entry;
-	struct vm_object *m_object;
-
-	/* used to link to  vm_object->vo_mappings */
-	struct queue_entry m_object_entry;
-
-	enum vm_prot m_prot;
-	/* offset in bytes to the start of the object data that was mapped */
-	off_t m_object_offset;
-};
-
-struct vm_region {
-	struct object vr_base;
-	struct vm_region_entry vr_entry;
-
-	char vr_name[VM_REGION_NAME_MAX];
-
-	/* btree of struct vm_region_entry.
-	 * sibling entries cannot overlap each other, and child entries must
-	 * be entirely contained within their immediate parent entry. */
-	struct btree vr_entries;
-
-	/* memory protection restriction mask.
-	 * any mapping in this region, or any of its children, cannot use
-	 * protection flags that are not set in this mask.
-	 * for example, if VM_PROT_EXEC is /not/ set here, no mapping
-	 * can be created in this region or any child region with VM_PROT_EXEC
-	 * set. */
-	enum vm_prot vr_prot;
-
-	/* the physical address space in which mappings in this region (and
-	 * its children) are created */
-	pmap_t vr_pmap;
-};
-
-extern kern_status_t vm_region_type_init(void);
-
-/* create a new vm-region, optionally within a parent region.
- * `offset` is the byte offset within the parent region where the new region
- * should start.
- * if no parent is specified, `offset` is the absolute virtual address of the
- * start of the region.
- * in both cases, `len` is the length of the new region in bytes. */
-extern kern_status_t vm_region_create(
-	struct vm_region *parent,
-	const char *name,
-	off_t offset,
-	size_t len,
-	enum vm_prot prot,
-	struct vm_region **out);
-
-/* map a vm-object into a vm-region.
- * [region_offset,length] must fall within exactly one region, and cannot span
- * multiple sibling regions.
- * if [region_offset,length] falls within a child region, the map operation
- * will be transparently redirected to the relevant region.
- * `prot` must be allowed both by the region into which the mapping is being
- * created AND the vm-object being mapped. */
-extern kern_status_t vm_region_map_object(
-	struct vm_region *region,
-	off_t region_offset,
-	struct vm_object *object,
-	off_t object_offset,
-	size_t length,
-	enum vm_prot prot,
-	virt_addr_t *out);
-
-/* find the mapping corresponding to the given virtual address, and page-in the
- * necessary vm_page to allow the memory access to succeed. if the relevant
- * vm-object page hasn't been allocated yet, it will be allocated here. */
-extern kern_status_t vm_region_demand_map(
-	struct vm_region *region,
-	virt_addr_t addr,
-	enum pmap_fault_flags flags);
-
-/* get the absolute base virtual address of a region within its
- * parent/ancestors. */
-extern virt_addr_t vm_region_get_base_address(const struct vm_region *region);
-
-extern void vm_region_dump(struct vm_region *region, int depth);
-
-DEFINE_OBJECT_LOCK_FUNCTION(vm_region, vr_base)
-
-#endif