meta: rename kernel to Magenta

whether a mapping is private or shared determines how the mapping is handled
when a task is duplicated.

CMakeLists.txt

@@ -1,5 +1,5 @@
-cmake_minimum_required(VERSION 3.13)
-project(mango C ASM)
+cmake_minimum_required(VERSION 3.31)
+project(magenta C ASM)
 
 if (NOT BUILD_TOOLS_DIR)
 	message(FATAL_ERROR "No build tools directory specified. Please run build.sh")
@@ -9,8 +9,8 @@ set(CMAKE_C_STANDARD 17)
 
 set(kernel_arch x86_64)
 
-set(kernel_name "Mango")
-set(kernel_exe_name "mango_kernel")
+set(kernel_name "Magenta")
+set(kernel_exe_name "magenta_kernel")
 
 set(generic_src_dirs ds init kernel libc sched util vm syscall)
 set(kernel_sources "")
@@ -28,8 +28,6 @@ file(GLOB_RECURSE arch_sources_c arch/${kernel_arch}/*.c)
 file(GLOB_RECURSE arch_sources_asm arch/${kernel_arch}/*.S)
 file(GLOB_RECURSE arch_headers arch/${kernel_arch}/*.h)
 
-set_property(SOURCE ${arch_sources_asm} PROPERTY LANGUAGE C)
-
 add_executable(${kernel_exe_name}
 	${kernel_sources}
 	${kernel_headers}
@@ -40,7 +38,7 @@ add_executable(${kernel_exe_name}
 target_include_directories(${kernel_exe_name} PRIVATE
 	include
 	libc/include
-	libmango/include
+	libmagenta/include
 	arch/${kernel_arch}/include)
 target_compile_options(${kernel_exe_name} PRIVATE
 	-nostdlib -ffreestanding

README

@@ -1,4 +1,4 @@
-Mango
+Magenta
 =====
 
 It's a kernel!

arch/user/include/mango/machine/cpu.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_USER_CPU_H_
-#define MANGO_USER_CPU_H_
+#ifndef MAGENTA_USER_CPU_H_
+#define MAGENTA_USER_CPU_H_
 
 #ifdef __cplusplus
 extern "C" {

arch/user/include/mango/machine/hwlock.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_USER_HWLOCK_H_
-#define MANGO_USER_HWLOCK_H_
+#ifndef MAGENTA_USER_HWLOCK_H_
+#define MAGENTA_USER_HWLOCK_H_
 
 #define ML_HWLOCK_INIT (0)
 

arch/user/include/mango/machine/init.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_INIT_H_
-#define MANGO_X86_64_INIT_H_
+#ifndef MAGENTA_X86_64_INIT_H_
+#define MAGENTA_X86_64_INIT_H_
 
 #include <stddef.h>
 #include <stdint.h>

arch/user/include/mango/machine/irq.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_IRQ_H_
-#define MANGO_X86_64_IRQ_H_
+#ifndef MAGENTA_X86_64_IRQ_H_
+#define MAGENTA_X86_64_IRQ_H_
 
 
 #endif

arch/user/include/mango/machine/pmap.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_USER_PMAP_H_
-#define MANGO_USER_PMAP_H_
+#ifndef MAGENTA_USER_PMAP_H_
+#define MAGENTA_USER_PMAP_H_
 
 #include <stdint.h>
 

arch/user/include/mango/machine/vm.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_USER_VM_H_
-#define MANGO_USER_VM_H_
+#ifndef MAGENTA_USER_VM_H_
+#define MAGENTA_USER_VM_H_
 
 #include <stdint.h>
 

arch/x86_64/asm-offset.c

@@ -28,4 +28,6 @@
 static void __used common(void)
 {
 	OFFSET(THREAD_sp, struct thread, tr_sp);
+	OFFSET(THREAD_fsbase, struct thread, tr_ml.tr_fsbase);
+	OFFSET(THREAD_gsbase, struct thread, tr_ml.tr_gsbase);
 }

arch/x86_64/config.cmake

@@ -1,4 +1,5 @@
 target_compile_options(${kernel_exe_name} PRIVATE
 	-z max-page-size=0x1000 -m64 -mcmodel=large -mno-red-zone -mno-mmx
 	-mno-sse -mno-sse2 -D_64BIT -DBYTE_ORDER=1234)
-target_link_libraries(${kernel_exe_name} "-z max-page-size=0x1000" "-T ${CMAKE_CURRENT_SOURCE_DIR}/arch/x86_64/layout.ld")
+target_link_libraries(${kernel_exe_name} "-static -z max-page-size=0x1000" "-T ${CMAKE_CURRENT_SOURCE_DIR}/arch/x86_64/layout.ld")
+

arch/x86_64/include/arch/msr.h

@@ -3,6 +3,7 @@
 
 #include <stdint.h>
 
+#define MSR_FS_BASE        0xC0000100
 #define MSR_GS_BASE        0xC0000101
 #define MSR_KERNEL_GS_BASE 0xC0000102
 

arch/x86_64/include/arch/paging.h

@@ -1,8 +1,8 @@
 #ifndef ARCH_PAGING_H_
 #define ARCH_PAGING_H_
 
-#include <kernel/types.h>
 #include <kernel/compiler.h>
+#include <kernel/types.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -55,6 +55,7 @@ enum page_size {
    defined in pmap_ctrl.S */
 extern int gigabyte_pages(void);
 extern int enable_nx(void);
+extern int enable_wp(void);
 
 #ifdef __cplusplus
 }

arch/x86_64/include/kernel/machine/thread.h

@@ -3,6 +3,10 @@
 
 #include <kernel/sched.h>
 
+struct ml_thread {
+	virt_addr_t tr_gsbase, tr_fsbase;
+};
+
 struct ml_cpu_context;
 
 /* switch from one thread to another. the stack of the `to` thread must have
@@ -27,5 +31,24 @@ extern kern_status_t ml_thread_prepare_user_context(
 	virt_addr_t *kernel_sp,
 	const uintptr_t *args,
 	size_t nr_args);
+/* prepare the stack so that ml_thread_switch_user can jump to usermode
+ * with the specified register context */
+extern kern_status_t ml_thread_clone_user_context(
+	const struct ml_cpu_context *src_regs,
+	const struct ml_thread *src_ml,
+	struct ml_thread *dest_ml,
+	uintptr_t return_value,
+	virt_addr_t *kernel_sp);
+
+extern kern_status_t ml_thread_config_get(
+	struct thread *thread,
+	kern_config_key_t key,
+	void *out,
+	size_t max);
+extern kern_status_t ml_thread_config_set(
+	struct thread *thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
 
 #endif

arch/x86_64/init.c

@@ -20,6 +20,8 @@
 #include <kernel/util.h>
 #include <kernel/vm.h>
 
+#define HARDWARE_RNG
+
 #define PTR32(x)            ((void *)((uintptr_t)(x)))
 
 /* the physical address of the start of the memblock heap.
@@ -125,17 +127,21 @@ int ml_init(uintptr_t arg)
 		reserve_end = bsp.mod_base + bsp.mod_size;
 	}
 
+#if defined(HARDWARE_RNG)
 	if (ml_hwrng_available()) {
 		printk("cpu: ardware RNG available");
 		uint64_t seed = ml_hwrng_generate();
 		printk("cpu: RNG seed=%zx", seed);
 		init_random(seed);
 	} else {
+#endif
 		printk("cpu: hardware RNG unavailable");
 		uint64_t seed = 0xeddc4c8a679dc23f;
 		printk("cpu: RNG seed=%zx", seed);
 		init_random(seed);
+#if defined(HARDWARE_RNG)
 	}
+#endif
 
 	early_vm_init(reserve_end);
 

arch/x86_64/irq.c

@@ -8,6 +8,7 @@
 #include <kernel/panic.h>
 #include <kernel/sched.h>
 #include <kernel/syscall.h>
+#include <kernel/thread.h>
 #include <stddef.h>
 
 #define MAX_ISR_HANDLERS  16
@@ -166,6 +167,12 @@ int idt_load(struct idt_ptr *ptr)
 
 void isr_dispatch(struct ml_cpu_context *regs)
 {
+	struct thread *thr = get_current_thread();
+	if (thr) {
+		thr->tr_irqctx = regs;
+		put_current_thread(thr);
+	}
+
 	int_hook h = isr_handlers[regs->int_no];
 	if (h) {
 		h(regs);
@@ -188,6 +195,13 @@ void irq_dispatch(struct ml_cpu_context *regs)
 	end_charge_period();
 
 	irq_ack(regs->int_no);
+
+	struct thread *thr = get_current_thread();
+	if (thr) {
+		thr->tr_irqctx = regs;
+		put_current_thread(thr);
+	}
+
 	struct queue *hooks = &irq_hooks[regs->int_no - IRQ0];
 	queue_foreach(struct irq_hook, hook, hooks, irq_entry)
 	{
@@ -203,6 +217,12 @@ void irq_dispatch(struct ml_cpu_context *regs)
 
 void syscall_dispatch(struct ml_cpu_context *regs)
 {
+	struct thread *thr = get_current_thread();
+	if (thr) {
+		thr->tr_irqctx = regs;
+		put_current_thread(thr);
+	}
+
 	unsigned int sysid = regs->rax;
 	virt_addr_t syscall_impl = syscall_get_function(sysid);
 

arch/x86_64/panic.c

@@ -175,7 +175,7 @@ static bool read_stack_frame(
 	struct stack_frame *out)
 {
 	if (bp >= VM_PAGEMAP_BASE) {
-		*out = *(struct stack_frame *)out;
+		*out = *(struct stack_frame *)bp;
 		return true;
 	}
 
@@ -217,16 +217,18 @@ static void print_stack_trace(
 
 void ml_print_stack_trace(uintptr_t ip)
 {
-	struct task *task = current_task();
+	struct task *task = get_current_task();
 	struct address_space *space = task ? task->t_address_space : NULL;
 	uintptr_t bp;
 	asm volatile("mov %%rbp, %0" : "=r"(bp));
 	print_stack_trace(space, ip, bp);
+	put_current_task(task);
 }
 
 void ml_print_stack_trace_irq(struct ml_cpu_context *ctx)
 {
-	struct task *task = current_task();
+	struct task *task = get_current_task();
 	struct address_space *space = task ? task->t_address_space : NULL;
 	print_stack_trace(space, ctx->rip, ctx->rbp);
+	put_current_task(task);
 }

arch/x86_64/pmap.c

@@ -9,7 +9,7 @@
 #include <kernel/types.h>
 #include <kernel/vm-object.h>
 #include <kernel/vm.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 
 /* some helpful datasize constants */
 #define C_1GiB          0x40000000ULL
@@ -19,7 +19,7 @@
 #define PTR_TO_ENTRY(x) (((x) & ~VM_PAGE_MASK) | PTE_PRESENT | PTE_RW | PTE_USR)
 #define ENTRY_TO_PTR(x) ((x) & ~VM_PAGE_MASK)
 
-#define PFN(x)          ((x) >> VM_PAGE_SHIFT)
+#define PFN(x)          (((x) >> VM_PAGE_SHIFT) & 0xFFFFFFFFFF)
 
 static int can_use_gbpages = 0;
 static pmap_t kernel_pmap;
@@ -100,7 +100,7 @@ static void delete_ptab(phys_addr_t pt)
 		return;
 	}
 
-	pt &= ~VM_PAGE_MASK;
+	pt = ENTRY_TO_PTR(pt);
 	if (!pt) {
 		/* physical address of 0x0, nothing to delete */
 		return;
@@ -117,7 +117,7 @@ static void delete_pdir(phys_addr_t pd)
 		return;
 	}
 
-	pd &= ~0x1FFFFFULL;
+	pd &= ENTRY_TO_PTR(pd);
 	if (!pd) {
 		/* physical address of 0x0, nothing to delete */
 		return;
@@ -136,6 +136,81 @@ static void delete_pdir(phys_addr_t pd)
 	kfree(pdir);
 }
 
+kern_status_t pmap_get(
+	pmap_t pmap,
+	virt_addr_t pv,
+	pfn_t *out_pfn,
+	vm_prot_t *out_prot)
+{
+	unsigned int pml4t_index = BAD_INDEX, pdpt_index = BAD_INDEX,
+		     pd_index = BAD_INDEX, pt_index = BAD_INDEX;
+
+	pml4t_index = (pv >> 39) & 0x1FF;
+	pdpt_index = (pv >> 30) & 0x1FF;
+	pd_index = (pv >> 21) & 0x1FF;
+	pt_index = (pv >> 12) & 0x1FF;
+
+	/* 1. get PML4T (mandatory) */
+	struct pml4t *pml4t = vm_phys_to_virt(ENTRY_TO_PTR(pmap));
+	if (!pml4t) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	/* 2. traverse PML4T, get PDPT (mandatory) */
+	struct pdpt *pdpt = NULL;
+	if (!pml4t->p_entries[pml4t_index]) {
+		return KERN_NO_ENTRY;
+	} else {
+		pdpt = vm_phys_to_virt(
+			ENTRY_TO_PTR(pml4t->p_entries[pml4t_index]));
+	}
+
+	/* 3. traverse PDPT, get PDIR (optional, 4K and 2M only) */
+	struct pdir *pdir = NULL;
+	if (!pdpt->p_entries[pdpt_index]
+	    || pdpt->p_pages[pdpt_index] & PTE_PAGESIZE) {
+		return KERN_NO_ENTRY;
+	} else {
+		pdir = vm_phys_to_virt(
+			ENTRY_TO_PTR(pdpt->p_entries[pdpt_index]));
+	}
+
+	/* 4. traverse PDIR, get PTAB (optional, 4K only) */
+	struct ptab *ptab = NULL;
+	if (!pdir->p_entries[pd_index]
+	    || pdir->p_pages[pd_index] & PTE_PAGESIZE) {
+		/* entry is null, or points to a hugepage */
+		return KERN_NO_ENTRY;
+	} else {
+		ptab = vm_phys_to_virt(ENTRY_TO_PTR(pdir->p_entries[pd_index]));
+	}
+
+	uint64_t pte = ptab->p_pages[pt_index];
+	if (out_pfn) {
+		*out_pfn = PFN(pte);
+	}
+
+	if (out_prot) {
+		if (pte & PTE_PRESENT) {
+			*out_prot |= VM_PROT_USER;
+		}
+
+		if (pte & PTE_RW) {
+			*out_prot |= (VM_PROT_READ | VM_PROT_WRITE);
+		}
+
+		if (pte & PTE_USR) {
+			*out_prot |= VM_PROT_USER;
+		}
+
+		if (!(pte & PTE_NX)) {
+			*out_prot |= VM_PROT_EXEC;
+		}
+	}
+
+	return KERN_OK;
+}
+
 static kern_status_t do_pmap_add(
 	pmap_t pmap,
 	virt_addr_t pv,
@@ -241,6 +316,108 @@ static kern_status_t do_pmap_add(
 	return KERN_OK;
 }
 
+static kern_status_t do_pmap_remove(
+	pmap_t pmap,
+	virt_addr_t pv,
+	enum page_size size)
+{
+	if (pmap == PMAP_INVALID) {
+		return KERN_OK;
+	}
+
+	unsigned int pml4t_index = BAD_INDEX, pdpt_index = BAD_INDEX,
+		     pd_index = BAD_INDEX, pt_index = BAD_INDEX;
+
+	switch (size) {
+	case PS_4K:
+		pml4t_index = (pv >> 39) & 0x1FF;
+		pdpt_index = (pv >> 30) & 0x1FF;
+		pd_index = (pv >> 21) & 0x1FF;
+		pt_index = (pv >> 12) & 0x1FF;
+		break;
+	case PS_2M:
+		pml4t_index = (pv >> 39) & 0x1FF;
+		pdpt_index = (pv >> 30) & 0x1FF;
+		pd_index = (pv >> 21) & 0x1FF;
+		break;
+	case PS_1G:
+		if (!can_use_gbpages) {
+			return KERN_UNSUPPORTED;
+		}
+		pml4t_index = (pv >> 39) & 0x1FF;
+		pdpt_index = (pv >> 30) & 0x1FF;
+		break;
+	default:
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	/* 1. get PML4T (mandatory) */
+	struct pml4t *pml4t = vm_phys_to_virt(ENTRY_TO_PTR(pmap));
+	if (!pml4t) {
+		return KERN_OK;
+	}
+
+	/* 2. traverse PML4T, get PDPT (mandatory) */
+	struct pdpt *pdpt = NULL;
+	if (!pml4t->p_entries[pml4t_index]) {
+		return KERN_OK;
+	} else {
+		pdpt = vm_phys_to_virt(
+			ENTRY_TO_PTR(pml4t->p_entries[pml4t_index]));
+	}
+
+	/* if we're mapping a 1GiB page, we stop here */
+	if (size == PS_1G) {
+		if (pdpt->p_entries[pdpt_index] != 0) {
+			/* this slot points to a pdir, delete it.
+			   if this slot points to a hugepage, this does nothing
+			 */
+			delete_pdir(pdpt->p_entries[pdpt_index]);
+		}
+
+		pdpt->p_pages[pdpt_index] = 0;
+
+		return KERN_OK;
+	}
+
+	/* 3. traverse PDPT, get PDIR (optional, 4K and 2M only) */
+	struct pdir *pdir = NULL;
+	if (!pdpt->p_entries[pdpt_index]
+	    || pdpt->p_pages[pdpt_index] & PTE_PAGESIZE) {
+		/* entry is null, or points to a hugepage */
+		return KERN_OK;
+	} else {
+		pdir = vm_phys_to_virt(
+			ENTRY_TO_PTR(pdpt->p_entries[pdpt_index]));
+	}
+
+	/* if we're mapping a 2MiB page, we stop here */
+	if (size == PS_2M) {
+		if (pdir->p_entries[pd_index] != 0) {
+			/* this slot points to a ptab, delete it.
+			   if this slot points to a hugepage, this does nothing
+			 */
+			delete_ptab(pdir->p_entries[pd_index]);
+		}
+
+		pdir->p_pages[pd_index] = 0;
+		return KERN_OK;
+	}
+
+	/* 4. traverse PDIR, get PTAB (optional, 4K only) */
+	struct ptab *ptab = NULL;
+	if (!pdir->p_entries[pd_index]
+	    || pdir->p_pages[pd_index] & PTE_PAGESIZE) {
+		/* entry is null, or points to a hugepage */
+		return KERN_OK;
+	} else {
+		ptab = vm_phys_to_virt(ENTRY_TO_PTR(pdir->p_entries[pd_index]));
+	}
+
+	ptab->p_pages[pt_index] = 0;
+	return KERN_OK;
+}
+
 pmap_t get_kernel_pmap(void)
 {
 	return kernel_pmap;
@@ -253,6 +430,8 @@ void pmap_bootstrap(void)
 	       can_use_gbpages == 1 ? "en" : "dis");
 	enable_nx();
 	printk("pmap: NX protection enabled");
+	enable_wp();
+	printk("pmap: kernel-mode write protection enabled");
 
 	enum page_size hugepage = PS_2M;
 	if (can_use_gbpages) {
@@ -314,8 +493,46 @@ pmap_t pmap_create(void)
 	return pmap;
 }
 
+static void delete_pdpt(phys_addr_t pd)
+{
+	if (pd & PTE_PAGESIZE) {
+		/* this entry points to a hugepage, nothing to delete */
+		return;
+	}
+
+	pd &= ENTRY_TO_PTR(pd);
+	if (!pd) {
+		/* physical address of 0x0, nothing to delete */
+		return;
+	}
+
+	struct pdpt *pdpt = vm_phys_to_virt(ENTRY_TO_PTR(pd));
+	for (int i = 0; i < 512; i++) {
+		if (pdpt->p_pages[i] & PTE_PAGESIZE) {
+			/* this is a hugepage, there is nothing to delete */
+			continue;
+		}
+
+		if (!pdpt->p_entries[i]) {
+			continue;
+		}
+
+		delete_ptab(pdpt->p_entries[i]);
+	}
+
+	kfree(pdpt);
+}
+
 void pmap_destroy(pmap_t pmap)
 {
+	struct pml4t *pml4t = vm_phys_to_virt(ENTRY_TO_PTR(pmap));
+	for (unsigned int i = 0; i < 256; i++) {
+		if (pml4t->p_entries[i]) {
+			delete_pdpt(pml4t->p_entries[i]);
+		}
+	}
+
+	kfree(pml4t);
 }
 
 static void log_fault(virt_addr_t fault_addr, enum pmap_fault_flags flags)
@@ -358,11 +575,7 @@ kern_status_t pmap_handle_fault(
 {
 	// log_fault(fault_addr, flags);
 
-	if (flags & PMAP_FAULT_PRESENT) {
-		return KERN_FATAL_ERROR;
-	}
-
-	struct task *task = current_task();
+	struct task *task = get_current_task();
 	if (!task) {
 		return KERN_FATAL_ERROR;
 	}
@@ -373,7 +586,11 @@ kern_status_t pmap_handle_fault(
 	}
 
 	/* this must be called with `space` unlocked. */
-	return address_space_demand_map(space, fault_addr, flags);
+	kern_status_t status
+		= address_space_demand_map(space, fault_addr, flags);
+
+	put_current_task(task);
+	return status;
 }
 
 kern_status_t pmap_add(
@@ -404,10 +621,18 @@ kern_status_t pmap_add_block(
 
 kern_status_t pmap_remove(pmap_t pmap, virt_addr_t p)
 {
-	return KERN_OK;
+	return do_pmap_remove(pmap, p, PS_4K);
 }
 
 kern_status_t pmap_remove_range(pmap_t pmap, virt_addr_t p, size_t len)
 {
+	if (pmap == PMAP_INVALID) {
+		return KERN_OK;
+	}
+
+	for (size_t i = p; i < p + len; i += VM_PAGE_SIZE) {
+		pmap_remove(pmap, i);
+	}
+
 	return KERN_OK;
 }

arch/x86_64/pmap_ctrl.S

@@ -1,3 +1,11 @@
+	.global pmap_flush
+	.type pmap_flush, @function
+
+pmap_flush:
+	mov %cr3, %rax
+	mov %rax, %cr3
+	ret
+
 
 	.global pmap_switch
 	.type pmap_switch, @function
@@ -6,6 +14,7 @@ pmap_switch:
 	mov %rdi, %cr3
 	ret
 
+
 	.global gigabyte_pages
 	.type gigabyte_pages, @function
 
@@ -30,6 +39,7 @@ gigabyte_pages:
 	pop %rbp
 	ret
 
+
 	.global enable_nx
 	.type enable_nx, @function
 
@@ -40,3 +50,14 @@ enable_nx:
 	wrmsr
 
 	ret
+
+
+	.global enable_wp
+	.type enable_wp, @function
+
+enable_wp:
+	mov %cr0, %rax
+	or $0x10000, %rax
+	mov %rax, %cr0
+
+	ret

arch/x86_64/serial.c

@@ -28,6 +28,10 @@ void serial_send_byte(int device, char out)
 
 	outportb(device, out);
 
+	if (device == COM1) {
+		outportb(0xe9, out);
+	}
+
 	while (!transmit_empty(device)) {
 		_count++;
 	}
@@ -138,6 +142,7 @@ static struct console serialcon = {
 
 static int serial_irq1(void)
 {
+#if 0
 	if (serial_received(COM1)) {
 		unsigned char c = serial_recv_byte(COM1);
 		printk("serial: COM1 received %c", c);
@@ -147,6 +152,7 @@ static int serial_irq1(void)
 		unsigned char c = serial_recv_byte(COM3);
 		printk("serial: COM3 received %c", c);
 	}
+#endif
 
 	return 0;
 }

arch/x86_64/thread.c

@@ -1,5 +1,7 @@
+#include <arch/msr.h>
 #include <kernel/machine/cpu.h>
 #include <kernel/machine/thread.h>
+#include <kernel/thread.h>
 
 #define MAX_REG_ARGS 6
 #define REG_ARG_0    rdi
@@ -77,3 +79,79 @@ extern kern_status_t ml_thread_prepare_user_context(
 
 	return KERN_OK;
 }
+
+kern_status_t ml_thread_clone_user_context(
+	const struct ml_cpu_context *src_regs,
+	const struct ml_thread *src_ml,
+	struct ml_thread *dest_ml,
+	uintptr_t return_value,
+	virt_addr_t *kernel_sp)
+{
+	(*kernel_sp) -= sizeof(struct ml_cpu_context);
+
+	struct ml_cpu_context *regs = (struct ml_cpu_context *)(*kernel_sp);
+	memcpy(regs, src_regs, sizeof *regs);
+
+	regs->rax = return_value;
+
+	dest_ml->tr_fsbase = src_ml->tr_fsbase;
+	dest_ml->tr_gsbase = src_ml->tr_gsbase;
+
+	return KERN_OK;
+}
+
+kern_status_t ml_thread_config_get(
+	struct thread *thread,
+	kern_config_key_t key,
+	void *out,
+	size_t max)
+{
+	return KERN_OK;
+}
+
+kern_status_t ml_thread_config_set(
+	struct thread *thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len)
+{
+	struct thread *self = get_current_thread();
+	kern_status_t status = KERN_OK;
+
+	switch (key) {
+	case THREAD_CFG_FSBASE:
+		if (len != sizeof(thread->tr_ml.tr_fsbase)) {
+			status = KERN_INVALID_ARGUMENT;
+			break;
+		}
+
+		thread->tr_ml.tr_fsbase = *(virt_addr_t *)ptr;
+		if (thread == self) {
+			wrmsr(MSR_FS_BASE, thread->tr_ml.tr_fsbase);
+		}
+
+		break;
+	case THREAD_CFG_GSBASE:
+		if (len != sizeof(thread->tr_ml.tr_gsbase)) {
+			status = KERN_INVALID_ARGUMENT;
+			break;
+		}
+
+		thread->tr_ml.tr_gsbase = *(virt_addr_t *)ptr;
+		if (thread == self) {
+			/* we're in the kernel right now, so the user and kernel
+			 * gs-base registers are swapped. when we return to
+			 * usermode, this value will be swapped back into
+			 * the user gs-base register */
+			wrmsr(MSR_KERNEL_GS_BASE, thread->tr_ml.tr_gsbase);
+		}
+
+		break;
+	default:
+		status = KERN_INVALID_ARGUMENT;
+		break;
+	}
+
+	put_current_thread(self);
+	return status;
+}

arch/x86_64/thread_switch.S

@@ -13,6 +13,22 @@ ml_thread_switch:
 	push %rax
 	push %rcx
 	push %rdx
+
+	// set fs-base
+	mov $0xC0000100, %rcx
+	movq THREAD_fsbase(%rsi), %rax
+	movq THREAD_fsbase(%rsi), %rdx
+	shr $32, %rdx
+	wrmsr
+
+	// set (kernel) gs-base (it will be swapped back into user-gs-base at
+	// the end of this function)
+	mov $0xC0000102, %rcx
+	movq THREAD_gsbase(%rsi), %rax
+	movq THREAD_gsbase(%rsi), %rdx
+	shr $32, %rdx
+	wrmsr
+
 	push %rbx
 	pushq $0
 	push %rbp

arch/x86_64/toolchain.cmake

@@ -1,5 +1,5 @@
 # the name of the target operating system
-set(CMAKE_SYSTEM_NAME Mango)
+set(CMAKE_SYSTEM_NAME Magenta)
 
 # which compilers to use for C and C++
 set(CMAKE_C_COMPILER   x86_64-elf-gcc)

ds/ringbuffer.c

@@ -1,189 +0,0 @@
-#include <kernel/ringbuffer.h>
-#include <kernel/sched.h>
-#include <kernel/vm.h>
-
-size_t ringbuffer_unread(struct ringbuffer *ring_buffer)
-{
-	if (ring_buffer->r_read_ptr == ring_buffer->r_write_ptr) {
-		return 0;
-	}
-
-	if (ring_buffer->r_read_ptr > ring_buffer->r_write_ptr) {
-		return (ring_buffer->r_size - ring_buffer->r_read_ptr)
-		     + ring_buffer->r_write_ptr;
-	} else {
-		return (ring_buffer->r_write_ptr - ring_buffer->r_read_ptr);
-	}
-}
-
-size_t ringbuffer_avail(struct ringbuffer *ring_buffer)
-{
-	if (ring_buffer->r_read_ptr == ring_buffer->r_write_ptr) {
-		return ring_buffer->r_size - 1;
-	}
-
-	if (ring_buffer->r_read_ptr > ring_buffer->r_write_ptr) {
-		return ring_buffer->r_read_ptr - ring_buffer->r_write_ptr - 1;
-	} else {
-		return (ring_buffer->r_size - ring_buffer->r_write_ptr)
-		     + ring_buffer->r_read_ptr - 1;
-	}
-}
-
-static inline void increment_read(struct ringbuffer *ring_buffer)
-{
-	ring_buffer->r_read_ptr++;
-	if (ring_buffer->r_read_ptr == ring_buffer->r_size) {
-		ring_buffer->r_read_ptr = 0;
-	}
-}
-
-static inline void increment_write(struct ringbuffer *ring_buffer)
-{
-	ring_buffer->r_write_ptr++;
-	if (ring_buffer->r_write_ptr == ring_buffer->r_size) {
-		ring_buffer->r_write_ptr = 0;
-	}
-}
-
-size_t ringbuffer_read(
-	struct ringbuffer *ring_buffer,
-	size_t size,
-	void *p,
-	mango_flags_t flags)
-{
-	if (!ring_buffer) {
-		return 0;
-	}
-
-	unsigned char *buffer = p;
-	unsigned long lock_flags;
-	size_t collected = 0;
-
-	while (collected < size) {
-		spin_lock_irqsave(&ring_buffer->r_lock, &lock_flags);
-		while (ringbuffer_unread(ring_buffer) > 0 && collected < size) {
-			buffer[collected]
-				= ring_buffer
-			                  ->r_buffer[ring_buffer->r_read_ptr];
-			increment_read(ring_buffer);
-			collected++;
-		}
-
-		wakeup_queue(&ring_buffer->r_wait_writers);
-
-		if (flags & S_NOBLOCK) {
-			spin_unlock_irqrestore(
-				&ring_buffer->r_lock,
-				lock_flags);
-			break;
-		}
-
-		struct wait_item waiter;
-		wait_item_init(&waiter, current_thread());
-		thread_wait_begin(&waiter, &ring_buffer->r_wait_readers);
-		spin_unlock_irqrestore(&ring_buffer->r_lock, lock_flags);
-
-		if (collected < size) {
-			schedule(SCHED_NORMAL);
-		}
-
-		thread_wait_end(&waiter, &ring_buffer->r_wait_readers);
-	}
-
-	wakeup_queue(&ring_buffer->r_wait_writers);
-	return collected;
-}
-
-size_t ringbuffer_write(
-	struct ringbuffer *ring_buffer,
-	size_t size,
-	const void *p,
-	mango_flags_t flags)
-{
-	if (!ring_buffer || !size) {
-		return 0;
-	}
-
-	const unsigned char *buffer = p;
-	unsigned long lock_flags;
-	size_t written = 0;
-
-	while (written < size) {
-		spin_lock_irqsave(&ring_buffer->r_lock, &lock_flags);
-
-		while (ringbuffer_avail(ring_buffer) > 0 && written < size) {
-			ring_buffer->r_buffer[ring_buffer->r_write_ptr]
-				= buffer[written];
-			increment_write(ring_buffer);
-			written++;
-		}
-
-		wakeup_queue(&ring_buffer->r_wait_readers);
-
-		if (flags & S_NOBLOCK) {
-			spin_unlock_irqrestore(
-				&ring_buffer->r_lock,
-				lock_flags);
-			break;
-		}
-
-		struct wait_item waiter;
-		wait_item_init(&waiter, current_thread());
-		thread_wait_begin(&waiter, &ring_buffer->r_wait_writers);
-		spin_unlock_irqrestore(&ring_buffer->r_lock, lock_flags);
-
-		if (written < size) {
-			schedule(SCHED_NORMAL);
-		}
-
-		thread_wait_end(&waiter, &ring_buffer->r_wait_writers);
-	}
-
-	wakeup_queue(&ring_buffer->r_wait_readers);
-	return written;
-}
-
-struct ringbuffer *ringbuffer_create(size_t size)
-{
-	struct ringbuffer *out = kzalloc(sizeof(struct ringbuffer), VM_NORMAL);
-	if (!out) {
-		return NULL;
-	}
-
-	if (ringbuffer_init(out, size) != KERN_OK) {
-		kfree(out);
-		return NULL;
-	}
-
-	return out;
-}
-
-void ringbuffer_destroy(struct ringbuffer *ring_buffer)
-{
-	ringbuffer_deinit(ring_buffer);
-	kfree(ring_buffer);
-}
-
-kern_status_t ringbuffer_init(struct ringbuffer *buf, size_t size)
-{
-	buf->r_buffer = kmalloc(size, VM_NORMAL);
-	if (!buf->r_buffer) {
-		return KERN_NO_MEMORY;
-	}
-
-	buf->r_write_ptr = 0;
-	buf->r_read_ptr = 0;
-	buf->r_size = size;
-	buf->r_lock = SPIN_LOCK_INIT;
-
-	return KERN_OK;
-}
-
-kern_status_t ringbuffer_deinit(struct ringbuffer *buf)
-{
-	kfree(buf->r_buffer);
-	buf->r_buffer = NULL;
-
-	return KERN_OK;
-}

include/kernel/address-space.h

@@ -6,6 +6,7 @@
 #include <kernel/vm.h>
 
 #define ADDRESS_SPACE_COPY_ALL ((size_t)-1)
+#define ADDRESS_SPACE_F_
 
 struct address_space;
 struct vm_object;
@@ -17,8 +18,12 @@ struct vm_area {
 	 * will avoid the area, but fixed address mappings in this area
 	 * will succeed. */
 	struct vm_object *vma_object;
+	/* the address space that this vm_area is a part of */
+	struct address_space *vma_space;
 	/* used to link to vm_object->vo_mappings */
 	struct queue_entry vma_object_entry;
+	/* the memory control flags applied to this area */
+	vm_flags_t vma_flags;
 	/* the memory protection flags applied to this area */
 	vm_prot_t vma_prot;
 	/* offset in bytes to the start of the object data that was mapped */
@@ -80,6 +85,7 @@ extern kern_status_t address_space_map(
 	struct vm_object *object,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out);
 extern kern_status_t address_space_unmap(
@@ -102,6 +108,12 @@ extern kern_status_t address_space_release(
 	virt_addr_t base,
 	size_t length);
 
+/* duplicate all of the mappings in `src` within `dest. the duplication will use
+ * copy-on-write; page data will not be copied until it is written to. */
+extern kern_status_t address_space_duplicate(
+	struct address_space *dest,
+	struct address_space *src);
+
 extern bool address_space_validate_access(
 	struct address_space *region,
 	virt_addr_t base,
@@ -156,6 +168,12 @@ extern kern_status_t address_space_memmove_v(
 	size_t bytes_to_move,
 	size_t *nr_bytes_moved);
 
+extern kern_status_t address_space_translate(
+	struct address_space *space,
+	virt_addr_t in,
+	phys_addr_t *out,
+	unsigned long *irq_flags);
+
 void address_space_dump(struct address_space *region);
 
 DEFINE_OBJECT_LOCK_FUNCTION(address_space, s_base)

include/kernel/arg.h

@@ -1,7 +1,7 @@
 #ifndef KERNEL_ARG_H_
 #define KERNEL_ARG_H_
 
-#include <mango/types.h>
+#include <magenta/types.h>
 #include <stdbool.h>
 
 #define CMDLINE_MAX 4096

include/kernel/atomic.h

@@ -0,0 +1,66 @@
+#ifndef KERNEL_ATOMIC_H_
+#define KERNEL_ATOMIC_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+
+typedef int32_t atomic_t;
+
+/* load and return the value pointed to by `v` */
+static inline atomic_t atomic_load(atomic_t *v)
+{
+	return __atomic_load_n(v, __ATOMIC_ACQUIRE);
+}
+/* store the value `v` to the pointer `dest` */
+static inline void atomic_store(atomic_t *dest, atomic_t v)
+{
+	__atomic_store_n(dest, v, __ATOMIC_ACQUIRE);
+}
+/* store the value `v` to the pointer `dest`, and return the value previously
+ * stored at `dest` */
+static inline atomic_t atomic_exchange(atomic_t *dest, atomic_t v)
+{
+	return __atomic_exchange_n(dest, v, __ATOMIC_ACQUIRE);
+}
+/* compare the contents of `ptr` to the contents of `expected`.
+ * if they match, store the value `desired` to the pointer `ptr` and return
+ * true. if the do NOT match, store the value `*ptr` to the pointer `desired`
+ * and return false.
+ */
+static inline bool atomic_compare_exchange(
+	atomic_t *ptr,
+	atomic_t *expected,
+	atomic_t desired)
+{
+	return __atomic_compare_exchange_n(
+		ptr,
+		expected,
+		desired,
+		false,
+		__ATOMIC_ACQUIRE,
+		__ATOMIC_ACQUIRE);
+}
+
+/* perform the operation *ptr += val, and return the result */
+static inline atomic_t atomic_add_fetch(atomic_t *ptr, atomic_t val)
+{
+	return __atomic_add_fetch(ptr, val, __ATOMIC_ACQUIRE);
+}
+/* perform the operation *ptr -= val, and return the result */
+static inline atomic_t atomic_sub_fetch(atomic_t *ptr, atomic_t val)
+{
+	return __atomic_sub_fetch(ptr, val, __ATOMIC_ACQUIRE);
+}
+
+/* perform the operation *ptr += val, and return the previous value of *ptr */
+static inline atomic_t atomic_fetch_add(atomic_t *ptr, atomic_t val)
+{
+	return __atomic_fetch_add(ptr, val, __ATOMIC_ACQUIRE);
+}
+/* perform the operation *ptr -= val, and return the previous value of *ptr */
+static inline atomic_t atomic_fetch_sub(atomic_t *ptr, atomic_t val)
+{
+	return __atomic_fetch_sub(ptr, val, __ATOMIC_ACQUIRE);
+}
+
+#endif

include/kernel/bsp.h

@@ -2,7 +2,7 @@
 #define KERNEL_BSP_H_
 
 #include <kernel/compiler.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 #include <kernel/types.h>
 #include <stddef.h>
 #include <stdint.h>

include/kernel/console.h

@@ -16,8 +16,8 @@
 */
 #include <kernel/locks.h>
 #include <kernel/queue.h>
-#include <mango/status.h>
-#include <mango/types.h>
+#include <magenta/status.h>
+#include <magenta/types.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/equeue.h

@@ -4,7 +4,7 @@
 #include <kernel/locks.h>
 #include <kernel/object.h>
 #include <kernel/sched.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 #define EQUEUE_PACKET_MAX 100
 

include/kernel/fb.h

@@ -1,49 +0,0 @@
-#ifndef KERNEL_FB_H_
-#define KERNEL_FB_H_
-
-#include <stdint.h>
-
-enum framebuffer_flags {
-	FB_MODE_RGB = 0x01u,
-	FB_MODE_VGATEXT = 0x02u,
-	FB_MODE_PALETTE = 0x04u,
-};
-
-struct framebuffer_bitfield {
-	uint32_t b_offset;
-	uint16_t b_length;
-};
-
-struct framebuffer_varinfo {
-	enum framebuffer_flags fb_flags;
-
-	uint32_t fb_xres;
-	uint32_t fb_yres;
-	uint32_t fb_bpp;
-	uint32_t fb_stride;
-
-	union {
-		struct {
-			uint32_t fb_xcells;
-			uint32_t fb_ycells;
-		};
-
-		struct {
-			struct framebuffer_bitfield fb_red;
-			struct framebuffer_bitfield fb_green;
-			struct framebuffer_bitfield fb_blue;
-			struct framebuffer_bitfield fb_alpha;
-		};
-
-		struct {
-			uintptr_t fb_palette_addr;
-			uint16_t fb_palette_nr_colours;
-		};
-	};
-};
-
-struct framebuffer_fixedinfo {
-	uint64_t fb_baseptr;
-};
-
-#endif

include/kernel/flags.h

@@ -1,11 +0,0 @@
-#ifndef KERNEL_FLAGS_H_
-#define KERNEL_FLAGS_H_
-
-#include <stdint.h>
-
-typedef enum {
-	S_NORMAL = 0x00u,
-	S_NOBLOCK = 0x01u,
-} mango_flags_t;
-
-#endif

include/kernel/futex.h

@@ -0,0 +1,34 @@
+#ifndef KERNEL_FUTEX_H_
+#define KERNEL_FUTEX_H_
+
+#include <kernel/btree.h>
+#include <kernel/wait.h>
+#include <magenta/types.h>
+
+struct task;
+struct address_space;
+
+typedef uintptr_t futex_key_t;
+
+struct futex {
+	struct btree_node f_node;
+
+	futex_key_t f_key;
+	struct waitqueue f_waiters;
+};
+
+extern kern_status_t futex_init(void);
+extern kern_status_t futex_get(
+	kern_futex_t *futex,
+	futex_key_t *out,
+	unsigned int flags);
+extern kern_status_t futex_wait(
+	futex_key_t futex,
+	kern_futex_t new_val,
+	unsigned int flags);
+extern kern_status_t futex_wake(
+	futex_key_t futex,
+	size_t nwaiters,
+	unsigned int flags);
+
+#endif

include/kernel/handle.h

@@ -2,8 +2,8 @@
 #define KERNEL_HANDLE_H_
 
 #include <kernel/bitmap.h>
-#include <mango/status.h>
-#include <mango/types.h>
+#include <magenta/status.h>
+#include <magenta/types.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -44,9 +44,13 @@ struct handle_table {
 
 extern struct handle_table *handle_table_create(void);
 extern void handle_table_destroy(struct handle_table *tab);
+extern kern_status_t handle_table_duplicate(
+	struct handle_table *src,
+	struct handle_table **dest);
 
 extern kern_status_t handle_table_alloc_handle(
 	struct handle_table *tab,
+	kern_handle_t value,
 	struct handle **out_slot,
 	kern_handle_t *out_handle);
 extern kern_status_t handle_table_free_handle(

include/kernel/input.h

@@ -1,184 +0,0 @@
-#ifndef KERNEL_INPUT_H_
-#define KERNEL_INPUT_H_
-
-#include <stdint.h>
-#include <kernel/queue.h>
-#include <mango/status.h>
-
-enum input_event_hook_flags {
-	INPUT_HOOK_SQUASH_EVENT = 0x01u,
-};
-
-struct device;
-
-enum input_event_type {
-	INPUT_TYPE_UNKNOWN = 0x00u,
-	INPUT_TYPE_KEY     = 0x01u,
-	INPUT_TYPE_MOTION  = 0x02u,
-};
-
-enum input_event_motion_type {
-	INPUT_MOTION_TYPE_MOUSE  = 0x01u,
-	INPUT_MOTION_TYPE_SCROLL = 0x02u,
-};
-
-enum input_keycode {
-	KEY_UNKNOWN                = 0x00u,
-	KEY_A                      = 0x01u,
-	KEY_B                      = 0x02u,
-	KEY_C                      = 0x03u,
-	KEY_D                      = 0x04u,
-	KEY_E                      = 0x05u,
-	KEY_F                      = 0x06u,
-	KEY_G                      = 0x07u,
-	KEY_H                      = 0x08u,
-	KEY_I                      = 0x09u,
-	KEY_J                      = 0x0Au,
-	KEY_K                      = 0x0Bu,
-	KEY_L                      = 0x0Cu,
-	KEY_M                      = 0x0Du,
-	KEY_N                      = 0x0Eu,
-	KEY_O                      = 0x0Fu,
-	KEY_P                      = 0x10u,
-	KEY_Q                      = 0x11u,
-	KEY_R                      = 0x12u,
-	KEY_S                      = 0x13u,
-	KEY_T                      = 0x14u,
-	KEY_U                      = 0x15u,
-	KEY_V                      = 0x16u,
-	KEY_W                      = 0x17u,
-	KEY_X                      = 0x18u,
-	KEY_Y                      = 0x19u,
-	KEY_Z                      = 0x1Au,
-	KEY_KEY_1                  = 0x1Bu,
-	KEY_KEY_2                  = 0x1Cu,
-	KEY_KEY_3                  = 0x1Du,
-	KEY_KEY_4                  = 0x1Eu,
-	KEY_KEY_5                  = 0x1Fu,
-	KEY_KEY_6                  = 0x20u,
-	KEY_KEY_7                  = 0x21u,
-	KEY_KEY_8                  = 0x22u,
-	KEY_KEY_9                  = 0x23u,
-	KEY_KEY_0                  = 0x24u,
-	KEY_ENTER                  = 0x25u,
-	KEY_ESCAPE                 = 0x26u,
-	KEY_BACKSPACE              = 0x27u,
-	KEY_TAB                    = 0x28u,
-	KEY_SPACE                  = 0x29u,
-	KEY_MINUS                  = 0x2Au,
-	KEY_EQUALS                 = 0x2Bu,
-	KEY_LEFT_BRACE             = 0x2Cu,
-	KEY_RIGHT_BRACE            = 0x2Du,
-	KEY_BACKSLASH              = 0x2Eu,
-	KEY_NON_US_HASH            = 0x2Fu,
-	KEY_SEMICOLON              = 0x30u,
-	KEY_APOSTROPHE             = 0x31u,
-	KEY_GRAVE_ACCENT           = 0x32u,
-	KEY_COMMA                  = 0x33u,
-	KEY_DOT                    = 0x34u,
-	KEY_SLASH                  = 0x35u,
-	KEY_CAPS_LOCK              = 0x36u,
-	KEY_F1                     = 0x37u,
-	KEY_F2                     = 0x38u,
-	KEY_F3                     = 0x39u,
-	KEY_F4                     = 0x3Au,
-	KEY_F5                     = 0x3Bu,
-	KEY_F6                     = 0x3Cu,
-	KEY_F7                     = 0x3Du,
-	KEY_F8                     = 0x3Eu,
-	KEY_F9                     = 0x3Fu,
-	KEY_F10                    = 0x40u,
-	KEY_F11                    = 0x41u,
-	KEY_F12                    = 0x42u,
-	KEY_PRINT_SCREEN           = 0x43u,
-	KEY_SCROLL_LOCK            = 0x44u,
-	KEY_PAUSE                  = 0x45u,
-	KEY_INSERT                 = 0x46u,
-	KEY_HOME                   = 0x47u,
-	KEY_PAGE_UP                = 0x48u,
-	KEY_DELETE                 = 0x49u,
-	KEY_END                    = 0x4Au,
-	KEY_PAGE_DOWN              = 0x4Bu,
-	KEY_RIGHT                  = 0x4Cu,
-	KEY_LEFT                   = 0x4Du,
-	KEY_DOWN                   = 0x4Eu,
-	KEY_UP                     = 0x4Fu,
-	KEY_NUM_LOCK               = 0x50u,
-	KEY_KEYPAD_SLASH           = 0x51u,
-	KEY_KEYPAD_ASTERISK        = 0x52u,
-	KEY_KEYPAD_MINUS           = 0x53u,
-	KEY_KEYPAD_PLUS            = 0x54u,
-	KEY_KEYPAD_ENTER           = 0x55u,
-	KEY_KEYPAD_1               = 0x56u,
-	KEY_KEYPAD_2               = 0x57u,
-	KEY_KEYPAD_3               = 0x58u,
-	KEY_KEYPAD_4               = 0x59u,
-	KEY_KEYPAD_5               = 0x5Au,
-	KEY_KEYPAD_6               = 0x5Bu,
-	KEY_KEYPAD_7               = 0x5Cu,
-	KEY_KEYPAD_8               = 0x5Du,
-	KEY_KEYPAD_9               = 0x5Eu,
-	KEY_KEYPAD_0               = 0x5Fu,
-	KEY_KEYPAD_DOT             = 0x60u,
-	KEY_NON_US_BACKSLASH       = 0x61u,
-	KEY_KEYPAD_EQUALS          = 0x62u,
-	KEY_MENU                   = 0x63u,
-	KEY_LEFT_CTRL              = 0x64u,
-	KEY_LEFT_SHIFT             = 0x65u,
-	KEY_LEFT_ALT               = 0x66u,
-	KEY_LEFT_META              = 0x67u,
-	KEY_RIGHT_CTRL             = 0x68u,
-	KEY_RIGHT_SHIFT            = 0x69u,
-	KEY_RIGHT_ALT              = 0x6Au,
-	KEY_RIGHT_META             = 0x6Bu,
-	KEY_MEDIA_MUTE             = 0x6Cu,
-	KEY_MEDIA_VOLUME_INCREMENT = 0x6Du,
-	KEY_MEDIA_VOLUME_DECREMENT = 0x6Eu,
-};
-
-enum input_key_state {
-	INPUT_KEYSTATE_DOWN = 0x00u,
-	INPUT_KEYSTATE_UP   = 0x01u,
-};
-
-enum input_button {
-	INPUT_BUTTON_MOUSE_LEFT    = 0x00u,
-	INPUT_BUTTON_MOUSE_MIDDLE  = 0x01u,
-	INPUT_BUTTON_MOUSE_RIGHT   = 0x02u,
-	INPUT_BUTTON_MOUSE_BACK    = 0x03u,
-	INPUT_BUTTON_MOUSE_FORWARD = 0x04u,
-};
-
-enum input_button_state {
-	INPUT_BUTTON_DOWN = 0x00u,
-	INPUT_BUTTON_UP   = 0x01u,
-};
-
-struct input_event {
-	enum input_event_type ev_type;
-	union {
-		struct {
-			enum input_event_motion_type type;
-			int16_t movement_x;
-			int16_t movement_y;
-		} ev_motion;
-
-		struct {
-			enum input_button button;
-			enum input_button_state state;
-		} ev_button;
-
-		struct {
-			enum input_keycode key;
-			enum input_key_state state;
-		} ev_key;
-	};
-};
-
-struct input_event_hook {
-	void(*hook_callback)(struct device *, struct input_event *, enum input_event_hook_flags *, void *);
-	void *hook_arg;
-	struct queue_entry hook_head;
-};
-
-#endif

include/kernel/iovec.h

@@ -1,7 +1,7 @@
 #ifndef KERNEL_IOVEC_H_
 #define KERNEL_IOVEC_H_
 
-#include <mango/types.h>
+#include <magenta/types.h>
 #include <stddef.h>
 
 struct address_space;

include/kernel/msg.h

@@ -3,8 +3,8 @@
 
 #include <kernel/btree.h>
 #include <kernel/locks.h>
-#include <mango/status.h>
-#include <mango/types.h>
+#include <magenta/status.h>
+#include <magenta/types.h>
 
 struct port;
 struct thread;
@@ -13,6 +13,7 @@ enum kmsg_status {
 	KMSG_WAIT_RECEIVE,
 	KMSG_WAIT_REPLY,
 	KMSG_REPLY_SENT,
+	KMSG_ASYNC,
 };
 
 struct msg {
@@ -20,10 +21,38 @@ struct msg {
 	enum kmsg_status msg_status;
 	struct btree_node msg_node;
 	msgid_t msg_id;
-	kern_status_t msg_result;
+	union {
+		/* only valid for asynchronous messages (msg_status ==
+		 * KMSG_ASYNC) */
+		struct {
+			koid_t msg_sender_port_id;
+			tid_t msg_sender_thread_id;
+		};
+		/* only valid for synchronous messages (msg_status !=
+		 * KMSG_ASYNC) */
+		struct {
 			struct port *msg_sender_port;
 			struct thread *msg_sender_thread;
+		};
+	};
+	kern_status_t msg_result;
+	kern_msg_type_t msg_type;
+
+	union {
+		/* msg_type = KERN_MSG_TYPE_DATA */
+		struct {
 			kern_msg_t msg_req, msg_resp;
 		};
 
+		/* msg_type = KERN_MSG_TYPE_EVENT */
+		struct {
+			kern_msg_event_type_t msg_event;
+		};
+	};
+};
+
+extern void msg_init(void);
+extern struct msg *msg_alloc(void);
+extern void msg_free(struct msg *msg);
+
 #endif

include/kernel/object.h

@@ -1,11 +1,11 @@
 #ifndef KERNEL_OBJECT_H_
 #define KERNEL_OBJECT_H_
 
-#include <kernel/flags.h>
+#include <kernel/atomic.h>
 #include <kernel/locks.h>
 #include <kernel/vm.h>
 #include <kernel/wait.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 #include <stddef.h>
 
 #ifdef __cplusplus
@@ -65,7 +65,7 @@ extern "C" {
 #define OBJECT_PATH_MAX 256
 
 #define OBJECT_CAST(to_type, to_type_member, p)                                \
-	((to_type *)((uintptr_t)p) - offsetof(to_type, to_type_member))
+	((to_type *)(((uintptr_t)p) - offsetof(to_type, to_type_member)))
 #define OBJECT_C_CAST(c_type, c_type_member, obj_type, objp)                   \
 	OBJECT_IS_TYPE(objp, obj_type)                                         \
 	? OBJECT_CAST(c_type, c_type_member, (objp)) : NULL
@@ -79,10 +79,7 @@ enum object_type_flags {
 };
 
 struct object_ops {
-	kern_status_t (*destroy)(struct object *obj, struct queue *q);
-	kern_status_t (*destroy_recurse)(
-		struct queue_entry *entry,
-		struct object **out);
+	kern_status_t (*destroy)(struct object *obj);
 };
 
 struct object_type {
@@ -101,8 +98,7 @@ struct object {
 	struct object_type *ob_type;
 	spin_lock_t ob_lock;
 	uint32_t ob_signals;
-	unsigned int ob_refcount;
-	unsigned int ob_handles;
+	atomic_t ob_refcount;
 	struct queue_entry ob_list;
 	struct waitqueue ob_wq;
 } __aligned(sizeof(long));
@@ -114,8 +110,6 @@ extern kern_status_t object_type_unregister(struct object_type *p);
 extern struct object *object_create(struct object_type *type);
 extern struct object *object_ref(struct object *obj);
 extern void object_unref(struct object *obj);
-extern void object_add_handle(struct object *obj);
-extern void object_remove_handle(struct object *obj);
 extern void object_lock(struct object *obj);
 extern void object_unlock(struct object *obj);
 extern void object_lock_irqsave(struct object *obj, unsigned long *flags);

include/kernel/percpu.h

@@ -1,7 +1,7 @@
 #ifndef KERNEL_PERCPU_H_
 #define KERNEL_PERCPU_H_
 
-#include <mango/status.h>
+#include <magenta/status.h>
 #include <kernel/compiler.h>
 #include <kernel/sched.h>
 

include/kernel/pmap.h

@@ -5,11 +5,10 @@
 
 #include <kernel/machine/pmap.h>
 #include <kernel/vm.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 #include <stddef.h>
 
 #define PMAP_INVALID ML_PMAP_INVALID
-#define PFN(x)       ((x) >> VM_PAGE_SHIFT)
 
 #ifdef __cplusplus
 extern "C" {
@@ -53,10 +52,19 @@ extern pmap_t pmap_create(void);
 extern void pmap_destroy(pmap_t pmap);
 extern void pmap_switch(pmap_t pmap);
 
+extern void pmap_flush(void);
+extern void pmap_flush_page(virt_addr_t p);
+extern void pmap_flush_range(virt_addr_t start, size_t length);
+
 extern kern_status_t pmap_handle_fault(
 	virt_addr_t fault_addr,
 	enum pmap_fault_flags flags);
 
+extern kern_status_t pmap_get(
+	pmap_t pmap,
+	virt_addr_t p,
+	pfn_t *out_pfn,
+	vm_prot_t *out_prot);
 extern kern_status_t pmap_add(
 	pmap_t pmap,
 	virt_addr_t p,

include/kernel/port.h

@@ -9,12 +9,6 @@ enum port_status {
 	PORT_OFFLINE = 0,
 	/* port is connected and ready to send messages */
 	PORT_READY,
-	/* port has sent a message, and is waiting for the remote to receive it
-	 */
-	PORT_SEND_BLOCKED,
-	/* port has sent a message, and the remote has received it. waiting for
-	 * the remote to reply. */
-	PORT_REPLY_BLOCKED,
 };
 
 struct port {

include/kernel/ringbuffer.h

@@ -1,45 +0,0 @@
-#ifndef KERNEL_RINGBUFFER_H_
-#define KERNEL_RINGBUFFER_H_
-
-#include <kernel/flags.h>
-#include <kernel/locks.h>
-#include <kernel/types.h>
-#include <kernel/wait.h>
-
-struct ringbuffer {
-	unsigned char *r_buffer;
-	size_t r_write_ptr;
-	size_t r_read_ptr;
-	size_t r_size;
-	spin_lock_t r_lock;
-	struct waitqueue r_wait_readers;
-	struct waitqueue r_wait_writers;
-};
-
-extern struct ringbuffer *ringbuffer_create(size_t size);
-extern void ringbuffer_destroy(struct ringbuffer *buf);
-
-extern kern_status_t ringbuffer_init(struct ringbuffer *buf, size_t size);
-extern kern_status_t ringbuffer_deinit(struct ringbuffer *buf);
-
-extern size_t ringbuffer_unread(struct ringbuffer *buf);
-extern size_t ringbuffer_avail(struct ringbuffer *buf);
-extern size_t ringbuffer_read(
-	struct ringbuffer *buf,
-	size_t size,
-	void *buffer,
-	mango_flags_t flags);
-extern size_t ringbuffer_write(
-	struct ringbuffer *buf,
-	size_t size,
-	const void *buffer,
-	mango_flags_t flags);
-
-/* TODO */
-// extern size_t ringbuffer_peek(struct ringbuffer *buf, size_t at, size_t size,
-// void *buffer); extern size_t ringbuffer_skip(struct ringbuffer *buf, size_t
-// count);
-
-extern int ringbuffer_write_would_block(struct ringbuffer *buf);
-
-#endif

include/kernel/sched.h

@@ -6,7 +6,7 @@
 #include <kernel/locks.h>
 #include <kernel/queue.h>
 #include <kernel/types.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 
 #define PRIO_MAX 32
 
@@ -76,8 +76,10 @@ extern struct runqueue *cpu_rq(unsigned int cpu);
 extern cycles_t default_quantum(void);
 
 extern bool need_resched(void);
-extern struct task *current_task(void);
-extern struct thread *current_thread(void);
+extern struct task *get_current_task(void);
+extern struct thread *get_current_thread(void);
+extern void put_current_task(struct task *task);
+extern void put_current_thread(struct thread *thread);
 
 extern struct runqueue *select_rq_for_thread(struct thread *thr);
 extern void schedule_thread_on_cpu(struct thread *thr);

include/kernel/syscall.h

@@ -5,8 +5,8 @@
 #include <kernel/handle.h>
 #include <kernel/task.h>
 #include <kernel/vm.h>
-#include <mango/status.h>
-#include <mango/syscall.h>
+#include <magenta/status.h>
+#include <magenta/syscall.h>
 
 #define validate_access(task, ptr, len, flags)                                 \
 	__validate_access(task, (const void *)ptr, len, flags)
@@ -42,6 +42,7 @@ extern kern_status_t sys_task_exit(int status);
 extern kern_status_t sys_task_self(kern_handle_t *out);
 extern kern_status_t sys_task_create(
 	kern_handle_t parent_handle,
+	task_flags_t flags,
 	const char *name,
 	size_t name_len,
 	kern_handle_t *out_task,
@@ -56,8 +57,33 @@ extern kern_status_t sys_task_create_thread(
 extern kern_status_t sys_task_get_address_space(
 	kern_handle_t task,
 	kern_handle_t *out);
+extern kern_status_t sys_task_config_get(
+	kern_handle_t task,
+	kern_config_key_t key,
+	void *ptr,
+	size_t len);
+extern kern_status_t sys_task_config_set(
+	kern_handle_t task,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+extern kern_status_t sys_task_duplicate(
+	kern_handle_t *out_task,
+	kern_handle_t *out_address_space);
 
+extern kern_status_t sys_thread_self(kern_handle_t *out);
 extern kern_status_t sys_thread_start(kern_handle_t thread);
+extern kern_status_t sys_thread_exit(void);
+extern kern_status_t sys_thread_config_get(
+	kern_handle_t thread,
+	kern_config_key_t key,
+	void *ptr,
+	size_t len);
+extern kern_status_t sys_thread_config_set(
+	kern_handle_t thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
 
 extern kern_status_t sys_vm_object_create(
 	const char *name,
@@ -103,6 +129,7 @@ extern kern_status_t sys_address_space_map(
 	kern_handle_t object,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out_base_address);
 extern kern_status_t sys_address_space_unmap(
@@ -121,9 +148,19 @@ extern kern_status_t sys_address_space_release(
 
 extern kern_status_t sys_kern_log(const char *s);
 extern kern_status_t sys_kern_handle_close(kern_handle_t handle);
-extern kern_status_t sys_kern_handle_duplicate(
+extern kern_status_t sys_kern_handle_transfer(
+	kern_handle_t src_task_handle,
+	kern_handle_t src_handle,
+	kern_handle_t dest_task_handle,
+	kern_handle_t dest_handle,
+	unsigned int mode,
+	kern_handle_t *out_handle);
+extern kern_status_t sys_kern_handle_control(
+	kern_handle_t task,
 	kern_handle_t handle,
-	kern_handle_t *out);
+	uint32_t set_mask,
+	uint32_t clear_mask,
+	uint32_t *out_flags);
 extern kern_status_t sys_kern_config_get(
 	kern_config_key_t key,
 	void *ptr,
@@ -169,11 +206,14 @@ extern kern_status_t sys_msg_write(
 extern kern_status_t sys_kern_object_wait(
 	kern_wait_item_t *items,
 	size_t nr_items);
+extern kern_status_t sys_kern_object_query(
+	kern_handle_t object,
+	kern_object_info_t *out);
 
 extern kern_status_t sys_vm_controller_create(kern_handle_t *out);
 extern kern_status_t sys_vm_controller_recv(
 	kern_handle_t ctrl,
-	equeue_packet_page_request_t *out);
+	equeue_packet_vm_request_t *out);
 extern kern_status_t sys_vm_controller_recv_async(
 	kern_handle_t ctrl,
 	kern_handle_t eq,
@@ -186,6 +226,14 @@ extern kern_status_t sys_vm_controller_create_object(
 	size_t data_len,
 	vm_prot_t prot,
 	kern_handle_t *out);
+extern kern_status_t sys_vm_controller_prepare_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	kern_handle_t *out_vmo);
+extern kern_status_t sys_vm_controller_finish_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key);
 extern kern_status_t sys_vm_controller_detach_object(
 	kern_handle_t ctrl,
 	kern_handle_t vmo);
@@ -197,6 +245,15 @@ extern kern_status_t sys_vm_controller_supply_pages(
 	off_t src_offset,
 	size_t count);
 
+extern kern_status_t sys_futex_wait(
+	kern_futex_t *futex,
+	kern_futex_t new_val,
+	unsigned int flags);
+extern kern_status_t sys_futex_wake(
+	kern_futex_t *futex,
+	unsigned int nr_waiters,
+	unsigned int flags);
+
 extern virt_addr_t syscall_get_function(unsigned int sysid);
 
 #endif

include/kernel/task.h

@@ -27,7 +27,7 @@ struct task {
 	struct address_space *t_address_space;
 	spin_lock_t t_handles_lock;
 	struct handle_table *t_handles;
-	struct btree b_channels;
+	struct btree t_channels, t_futex;
 
 	struct btree_node t_tasklist;
 	struct queue_entry t_child_entry;
@@ -39,7 +39,11 @@ struct task {
 
 extern struct task *task_alloc(void);
 extern struct task *task_cast(struct object *obj);
-extern struct task *task_create(const char *name, size_t name_len);
+extern struct task *task_create(
+	struct task *parent,
+	task_flags_t flags,
+	const char *name,
+	size_t name_len);
 static inline struct task *task_ref(struct task *task)
 {
 	return OBJECT_CAST(struct task, t_base, object_ref(&task->t_base));
@@ -48,6 +52,7 @@ static inline void task_unref(struct task *task)
 {
 	object_unref(&task->t_base);
 }
+extern void task_exit(int status);
 extern kern_status_t task_add_child(struct task *parent, struct task *child);
 extern kern_status_t task_add_channel(
 	struct task *task,
@@ -65,6 +70,16 @@ extern kern_status_t task_resolve_handle(
 	kern_handle_t handle,
 	struct object **out_obj,
 	handle_flags_t *out_flags);
+extern kern_status_t task_config_get(
+	struct task *task,
+	kern_config_key_t key,
+	void *out,
+	size_t max);
+extern kern_status_t task_config_set(
+	struct task *task,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
 extern kern_status_t task_close_handle(struct task *task, kern_handle_t handle);
 extern struct thread *task_create_thread(struct task *parent);
 extern struct task *kernel_task(void);

include/kernel/test.h

@@ -1,14 +0,0 @@
-#ifndef KERNEL_TEST_H_
-#define KERNEL_TEST_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern int run_all_tests(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

include/kernel/thread.h

@@ -1,12 +1,15 @@
 #ifndef KERNEL_THREAD_H_
 #define KERNEL_THREAD_H_
 
+#include <kernel/machine/thread.h>
 #include <kernel/msg.h>
 #include <kernel/object.h>
 #include <kernel/vm-controller.h>
 
 #define THREAD_KSTACK_ORDER VM_PAGE_4K
 
+struct ml_cpu_context;
+
 enum thread_state {
 	THREAD_READY = 1,
 	THREAD_SLEEPING = 2,
@@ -22,7 +25,7 @@ enum thread_flags {
 };
 
 struct thread {
-	struct object thr_base;
+	struct object tr_base;
 
 	enum thread_state tr_state;
 	enum thread_flags tr_flags;
@@ -38,15 +41,16 @@ struct thread {
 	virt_addr_t tr_ip, tr_sp, tr_bp;
 	virt_addr_t tr_cpu_user_sp, tr_cpu_kernel_sp;
 
+	/* only valid within an interrupt or syscall context */
+	struct ml_cpu_context *tr_irqctx;
+
+	struct ml_thread tr_ml;
 	struct runqueue *tr_rq;
-	struct msg tr_msg;
-	struct page_request tr_page_req;
 
 	struct queue_entry tr_parent_entry;
 	struct queue_entry tr_rqentry;
 
 	struct vm_page *tr_kstack;
-	struct vm_object *tr_ustack;
 };
 
 extern struct thread *thread_alloc(void);
@@ -58,10 +62,30 @@ extern kern_status_t thread_init_user(
 	virt_addr_t sp,
 	const uintptr_t *args,
 	size_t nr_args);
+extern kern_status_t thread_init_user_clone(
+	struct thread *thr,
+	const struct thread *src,
+	uintptr_t return_value);
 extern int thread_priority(struct thread *thr);
 extern void thread_awaken(struct thread *thr);
+extern void thread_exit(void);
+extern void thread_join(struct thread *thread, unsigned long *irq_flags);
+extern void thread_kill(struct thread *thread);
 extern void idle(void);
 extern struct thread *create_kernel_thread(void (*fn)(void));
 extern struct thread *create_idle_thread(void);
 
+extern kern_status_t thread_config_get(
+	struct thread *thread,
+	kern_config_key_t key,
+	void *out,
+	size_t max);
+extern kern_status_t thread_config_set(
+	struct thread *thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+
+DEFINE_OBJECT_LOCK_FUNCTION(thread, tr_base)
+
 #endif

include/kernel/types.h

@@ -1,7 +1,7 @@
 #ifndef KERNEL_TYPES_H_
 #define KERNEL_TYPES_H_
 
-#include <mango/types.h>
+#include <magenta/types.h>
 #include <stddef.h>
 #include <stdint.h>
 

include/kernel/util.h

@@ -1,7 +1,7 @@
 #ifndef KERNEL_UTIL_H_
 #define KERNEL_UTIL_H_
 
-#include <mango/types.h>
+#include <magenta/types.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>

include/kernel/vm-controller.h

@@ -3,42 +3,41 @@
 
 #include <kernel/locks.h>
 #include <kernel/object.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 struct thread;
 struct equeue;
 struct vm_object;
 
-enum page_request_status {
-	PAGE_REQUEST_PENDING = 0,
-	PAGE_REQUEST_IN_PROGRESS,
-	PAGE_REQUEST_COMPLETE,
+enum vm_request_status {
+	VM_REQUEST_PENDING = 0,
+	VM_REQUEST_IN_PROGRESS,
+	VM_REQUEST_COMPLETE,
+	VM_REQUEST_ASYNC,
 };
 
 struct vm_controller {
 	struct object vc_base;
-	/* tree of struct vm_objects bound to this controller, keyed with the
-	 * equeue_key_t specified when the object(s) were created. */
-	struct btree vc_objects;
-	/* tree of pending page requests */
+	/* tree of pending vm requests */
 	struct btree vc_requests;
-	/* the equeue to send async page requests to */
 	struct equeue *vc_eq;
 	equeue_key_t vc_eq_key;
-	/* the number of page requests queued with status PAGE_REQUEST_PENDING.
+	/* the number of page requests queued with status VM_REQUEST_PENDING.
 	 * used to assert/clear VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED */
 	size_t vc_requests_waiting;
 };
 
-struct page_request {
+struct vm_request {
 	uint64_t req_id;
 	unsigned int req_type;
-	enum page_request_status req_status;
+	enum vm_request_status req_status;
 	kern_status_t req_result;
 	spin_lock_t req_lock;
 	struct vm_object *req_object;
 	struct thread *req_sender;
+	/* this node is added to vm-controller vc_requests list */
 	struct btree_node req_node;
+	/* these values are used for VM_REQUEST_READ and VM_REQUEST_DIRTY */
 	off_t req_offset;
 	size_t req_length;
 };
@@ -50,7 +49,7 @@ extern struct vm_controller *vm_controller_create(void);
 
 extern kern_status_t vm_controller_recv(
 	struct vm_controller *ctrl,
-	equeue_packet_page_request_t *out);
+	equeue_packet_vm_request_t *out);
 extern kern_status_t vm_controller_recv_async(
 	struct vm_controller *ctrl,
 	struct equeue *eq,
@@ -64,6 +63,14 @@ extern kern_status_t vm_controller_create_object(
 	size_t data_len,
 	vm_prot_t prot,
 	struct vm_object **out);
+extern kern_status_t vm_controller_prepare_attach(
+	struct vm_controller *ctrl,
+	uint64_t req_id,
+	struct vm_object **out_vmo);
+extern kern_status_t vm_controller_finish_attach(
+	struct vm_controller *ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key);
 extern kern_status_t vm_controller_detach_object(
 	struct vm_controller *ctrl,
 	struct vm_object *vmo);
@@ -74,10 +81,16 @@ extern kern_status_t vm_controller_supply_pages(
 	struct vm_object *src,
 	off_t src_offset,
 	size_t count);
+extern void vm_controller_fulfill_requests(
+	struct vm_controller *ctrl,
+	equeue_key_t object,
+	off_t offset,
+	size_t length,
+	kern_status_t result);
 
 extern kern_status_t vm_controller_send_request(
 	struct vm_controller *ctrl,
-	struct page_request *req,
+	struct vm_request *req,
 	unsigned long *irq_flags);
 
 DEFINE_OBJECT_LOCK_FUNCTION(vm_controller, vc_base)

include/kernel/vm-object.h

@@ -14,19 +14,31 @@ enum vm_object_flags {
 	VMO_IN_PLACE = 0x01u,
 	/* this vm-object is/was attached to a vm-controller */
 	VMO_CONTROLLER = 0x02u,
+	/* when a vm-object is attached to a controller, and the ref-count of
+	 * the object falls to one (i.e. the last reference is the handle to
+	 * the object held by the server that created it), the object will
+	 * be detached, allowing the server to close the last handle to the
+	 * object and dispose of it. */
+	VMO_AUTO_DETACH = 0x04u,
+	/* this vmo is a duplicate of a vmo that is attached to a vm-controller.
+	 * the duplicate vmo is scheduled to be attached to the same controller,
+	 * but this won't actually happen until the controller is needed to
+	 * fulfill a page request. once the duplicate vmo has been attached to
+	 * the controller, this flag will be cleared. */
+	VMO_LAZY_ATTACH = 0x08u,
 
 	/* these flags are for use with vm_object_get_page */
 	/**************************************************/
 
 	/* if the relevant page hasn't been allocated yet, it will be allocated
 	 * and returned. if this flag isn't specified, NULL will be returned. */
-	VMO_ALLOCATE_MISSING_PAGE = 0x04u,
+	VMO_ALLOCATE_MISSING_PAGE = 0x0100u,
 	/* if the vm-object is attached to a vm-controller, and the relevant
 	 * page is uncommitted, send a request to the vm-controller to provide
 	 * the missing page. will result in the vm-object being unlocked and
 	 * the current thread sleeping until the request is fulfilled. the
 	 * vm-object will be re-locked before the function returns. */
-	VMO_REQUEST_MISSING_PAGE = 0x08u,
+	VMO_REQUEST_MISSING_PAGE = 0x0200u,
 };
 
 struct vm_object {
@@ -78,11 +90,33 @@ extern struct vm_object *vm_object_create_in_place(
 	size_t data_len,
 	vm_prot_t prot);
 
+/* create a copy-on-write duplicate of a vm-object */
+extern struct vm_object *vm_object_duplicate_cow(struct vm_object *vmo);
+/* attach a copy-on-write duplicate of a vm-object to the vm-controller that
+ * controlled the original vm-object */
+extern kern_status_t vm_object_attach_cow(
+	struct vm_object *vmo,
+	unsigned long *irq_flags);
+
+/* prefetch any missing pages in the specified range of a vm-object.
+ * any lazy-allocated pages will be allocated.
+ * any missing pages will be requested from the vm-controller, if one is
+ * attached. */
+extern kern_status_t vm_object_prefetch(
+	struct vm_object *vo,
+	off_t offset,
+	size_t length,
+	unsigned long *irq_flags);
+
 extern struct vm_page *vm_object_get_page(
 	struct vm_object *vo,
 	off_t offset,
 	enum vm_object_flags flags,
 	unsigned long *irq_flags);
+extern kern_status_t vm_object_put_page(
+	struct vm_object *vo,
+	off_t offset,
+	struct vm_page *pg);
 
 extern kern_status_t vm_object_read(
 	struct vm_object *vo,

include/kernel/vm.h

@@ -1,21 +1,20 @@
 #ifndef KERNEL_VM_H_
 #define KERNEL_VM_H_
 
+#include <kernel/atomic.h>
 #include <kernel/bitmap.h>
 #include <kernel/btree.h>
 #include <kernel/locks.h>
 #include <kernel/machine/vm.h>
 #include <kernel/queue.h>
 #include <kernel/types.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 #include <stddef.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-struct bcache;
-
 /* maximum number of NUMA nodes */
 #define VM_MAX_NODES            64
 /* maximum number of memory zones per node */
@@ -25,11 +24,6 @@ struct bcache;
 /* maximum number of sparse memory sectors */
 #define VM_MAX_SECTORS          8192
 
-/* maximum number of disk sectors that can be stored in a single
-   page. AKA the number of bits in the sector bitmap.
-   used by the block cache */
-#define VM_MAX_SECTORS_PER_PAGE 32
-
 #define VM_CHECK_ALIGN(p, mask) ((((p) & (mask)) == (p)) ? 1 : 0)
 
 #define VM_CACHE_INITIALISED(c) ((c)->c_obj_count != 0)
@@ -207,7 +201,6 @@ struct vm_page {
 	/* owner-specific pointer */
 	union {
 		struct vm_slab *p_slab;
-		struct bcache *p_bcache;
 		void *p_priv0;
 	};
 
@@ -218,8 +211,6 @@ struct vm_page {
 	       lists.
 	     - vm_object uses it to maintain a btree of allocated pages keyed
 	       by offset/size.
-	     - the block cache uses this to maintain a tree of pages keyed by
-	       block number.
 	*/
 	union {
 		struct queue_entry p_list;
@@ -231,15 +222,13 @@ struct vm_page {
 	};
 
 	union {
-		/* used by bcache when sector size is < page size. bitmap of
-		 * present/missing sectors */
-		DECLARE_BITMAP(p_blockbits, VM_MAX_SECTORS_PER_PAGE);
+		/* how many vm_areas reference this vm_page. if >0, the page is
+		 * subject to copy-on-write. */
+		atomic_t p_cow_ref;
 		uint32_t p_priv2;
 	};
 
 	union {
-		/* sector address, used by bcache */
-		sectors_t p_blockid;
 		/* offset of this page within the vm_object it is a part of */
 		off_t p_vmo_offset;
 

include/kernel/wait.h

@@ -40,6 +40,15 @@ extern void thread_wait_end_nosleep(
 	struct waitqueue *q);
 extern void wait_on_queue(struct waitqueue *q);
 extern void wakeup_queue(struct waitqueue *q);
+extern void wakeup_n(struct waitqueue *q, size_t n);
 extern void wakeup_one(struct waitqueue *q);
+static inline bool waitqueue_empty(struct waitqueue *wq)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&wq->wq_lock, &flags);
+	bool result = queue_empty(&wq->wq_waiters);
+	spin_unlock_irqrestore(&wq->wq_lock, flags);
+	return result;
+}
 
 #endif

init/main.c

@@ -5,7 +5,6 @@
 #include <kernel/cpu.h>
 #include <kernel/handle.h>
 #include <kernel/init.h>
-#include <kernel/input.h>
 #include <kernel/libc/stdio.h>
 #include <kernel/machine/init.h>
 #include <kernel/object.h>
@@ -14,7 +13,6 @@
 #include <kernel/printk.h>
 #include <kernel/sched.h>
 #include <kernel/task.h>
-#include <kernel/test.h>
 #include <kernel/thread.h>
 #include <kernel/vm-object.h>
 #include <stdint.h>
@@ -25,7 +23,7 @@ extern char __pstart[], __pend[];
 
 void print_kernel_banner(void)
 {
-	printk("Mango kernel version " BUILD_ID);
+	printk("Magenta kernel version " BUILD_ID);
 }
 
 static void hang(void)
@@ -46,8 +44,8 @@ static void hang(void)
 
 void background_thread(void)
 {
-	struct task *self = current_task();
-	struct thread *thread = current_thread();
+	struct task *self = get_current_task();
+	struct thread *thread = get_current_thread();
 	printk("background_thread() running on processor %u", this_cpu());
 	milli_sleep(1000);
 
@@ -78,6 +76,7 @@ void kernel_init(uintptr_t arg)
 
 	port_type_init();
 	channel_type_init();
+	msg_init();
 
 	struct boot_module bsp_image = {0};
 	bsp_get_location(&bsp_image);
@@ -109,7 +108,8 @@ void kernel_init(uintptr_t arg)
 	       bsp.bsp_trailer.bsp_exec_entry,
 	       bsp.bsp_vmo);
 
-	struct task *bootstrap_task = task_create("bootstrap", 9);
+	struct task *bootstrap_task
+		= task_create(NULL, TASK_F_DEFAULT, "bootstrap", 9);
 	tracek("created bootstrap task (pid=%u)", bootstrap_task->t_id);
 
 	status = bsp_launch_async(&bsp, bootstrap_task);

kernel/arg.c

@@ -1,7 +1,7 @@
 #include <kernel/arg.h>
 #include <kernel/libc/ctype.h>
 #include <kernel/libc/string.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 
 static char g_cmdline[CMDLINE_MAX + 1] = {0};
 

kernel/bsp.c

@@ -120,6 +120,7 @@ static kern_status_t map_executable_exec(
 		bsp->bsp_vmo,
 		text_foffset,
 		bsp->bsp_trailer.bsp_text_size,
+		VM_SHARED,
 		VM_PROT_READ | VM_PROT_EXEC | VM_PROT_USER,
 		&text_base);
 	if (status != KERN_OK) {
@@ -132,6 +133,7 @@ static kern_status_t map_executable_exec(
 		data,
 		data_foffset,
 		bsp->bsp_trailer.bsp_data_size,
+		VM_PRIVATE,
 		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
 		&data_base);
 	if (status != KERN_OK) {
@@ -165,6 +167,7 @@ kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
 		user_stack,
 		0,
 		BOOTSTRAP_STACK_SIZE,
+		VM_PRIVATE,
 		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
 		&stack_buffer);
 
@@ -178,6 +181,7 @@ kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
 		bsp->bsp_vmo,
 		0,
 		bsp->bsp_trailer.bsp_exec_offset,
+		VM_PRIVATE,
 		VM_PROT_READ | VM_PROT_USER,
 		&bsp_data_base);
 

kernel/channel.c

@@ -2,10 +2,11 @@
 #include <kernel/channel.h>
 #include <kernel/msg.h>
 #include <kernel/port.h>
+#include <kernel/printk.h>
 #include <kernel/task.h>
 #include <kernel/thread.h>
 #include <kernel/util.h>
-#include <mango/signal.h>
+#include <magenta/signal.h>
 
 #define CHANNEL_CAST(p) OBJECT_C_CAST(struct channel, c_base, &channel_type, p)
 
@@ -98,11 +99,17 @@ static struct msg *get_next_msg(
 	while (cur) {
 		struct msg *msg = BTREE_CONTAINER(struct msg, msg_node, cur);
 		spin_lock_irqsave(&msg->msg_lock, lock_flags);
-		if (msg->msg_status == KMSG_WAIT_RECEIVE) {
+		switch (msg->msg_status) {
+		case KMSG_WAIT_RECEIVE:
 			msg->msg_status = KMSG_WAIT_REPLY;
-			msg->msg_sender_port->p_status = PORT_REPLY_BLOCKED;
 			channel->c_msg_waiting--;
 			return msg;
+		case KMSG_ASYNC:
+			btree_delete(&channel->c_msg, &msg->msg_node);
+			channel->c_msg_waiting--;
+			return msg;
+		default:
+			break;
 		}
 
 		spin_unlock_irqrestore(&msg->msg_lock, *lock_flags);
@@ -132,7 +139,6 @@ extern kern_status_t channel_recv_msg(
 	kern_msg_t *out_msg,
 	unsigned long *irq_flags)
 {
-	struct thread *self = current_thread();
 	struct msg *msg = NULL;
 	unsigned long msg_lock_flags;
 
@@ -146,26 +152,24 @@ extern kern_status_t channel_recv_msg(
 			&channel->c_base,
 			CHANNEL_SIGNAL_MSG_RECEIVED);
 	}
-#if 0
-	wait_item_init(&waiter, self);
-	for (;;) {
-		thread_wait_begin(&waiter, &channel->c_wq);
-		msg = get_next_msg(channel, &msg_lock_flags);
-		if (msg) {
-			break;
-		}
-
-		object_unlock_irqrestore(&channel->c_base, *irq_flags);
-		schedule(SCHED_NORMAL);
-		object_lock_irqsave(&channel->c_base, irq_flags);
-	}
-	thread_wait_end(&waiter, &channel->c_wq);
-#endif
 
 	/* msg is now set to the next message to process */
 
+	if (msg->msg_type != KERN_MSG_TYPE_DATA) {
+		/* event messages are asynchronous */
+		out_msg->msg_id = msg->msg_id;
+		out_msg->msg_type = msg->msg_type;
+		out_msg->msg_event = msg->msg_event;
+		out_msg->msg_sender = msg->msg_sender_thread_id;
+		out_msg->msg_endpoint = msg->msg_sender_port_id;
+		spin_unlock_irqrestore(&msg->msg_lock, msg_lock_flags);
+		msg_free(msg);
+
+		return KERN_OK;
+	}
+
 	struct task *sender = msg->msg_sender_thread->tr_parent;
-	struct task *receiver = self->tr_parent;
+	struct task *receiver = get_current_task();
 
 	struct address_space *src = sender->t_address_space,
 			     *dst = receiver->t_address_space;
@@ -187,6 +191,7 @@ extern kern_status_t channel_recv_msg(
 
 	if (status != KERN_OK) {
 		kmsg_reply_error(msg, status, &msg_lock_flags);
+		put_current_task(receiver);
 		return status;
 	}
 
@@ -211,6 +216,7 @@ extern kern_status_t channel_recv_msg(
 		&receiver->t_handles_lock,
 		f);
 	address_space_unlock_pair_irqrestore(src, dst, f);
+	put_current_task(receiver);
 
 	if (status != KERN_OK) {
 		kmsg_reply_error(msg, status, &msg_lock_flags);
@@ -218,6 +224,7 @@ extern kern_status_t channel_recv_msg(
 	}
 
 	out_msg->msg_id = msg->msg_id;
+	out_msg->msg_type = msg->msg_type;
 	out_msg->msg_sender = msg->msg_sender_thread->tr_parent->t_id;
 	out_msg->msg_endpoint = msg->msg_sender_port->p_base.ob_id;
 
@@ -244,11 +251,10 @@ extern kern_status_t channel_reply_msg(
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	struct thread *self = current_thread();
 	/* the task that is about to receive the response */
 	struct task *receiver = msg->msg_sender_thread->tr_parent;
 	/* the task that is about to send the response */
-	struct task *sender = self->tr_parent;
+	struct task *sender = get_current_task();
 
 	struct address_space *src = sender->t_address_space,
 			     *dst = receiver->t_address_space;
@@ -269,6 +275,7 @@ extern kern_status_t channel_reply_msg(
 
 	if (status != KERN_OK) {
 		kmsg_reply_error(msg, status, &msg_lock_flags);
+		put_current_task(sender);
 		return status;
 	}
 
@@ -293,6 +300,7 @@ extern kern_status_t channel_reply_msg(
 		&receiver->t_handles_lock,
 		f);
 	address_space_unlock_pair_irqrestore(src, dst, f);
+	put_current_task(sender);
 
 	if (status != KERN_OK) {
 		kmsg_reply_error(msg, status, &msg_lock_flags);

kernel/futex.c

@@ -0,0 +1,231 @@
+#include <kernel/address-space.h>
+#include <kernel/futex.h>
+#include <kernel/sched.h>
+#include <kernel/task.h>
+#include <magenta/status.h>
+
+#define FUTEX_CREATE 0x40u
+
+static struct btree shared_futex_list = {0};
+static spin_lock_t shared_futex_list_lock = SPIN_LOCK_INIT;
+
+static struct vm_cache futex_cache = {
+	.c_name = "futex",
+	.c_obj_size = sizeof(struct futex),
+};
+
+BTREE_DEFINE_SIMPLE_INSERT(struct futex, f_node, f_key, put_futex)
+BTREE_DEFINE_SIMPLE_GET(struct futex, uintptr_t, f_node, f_key, get_futex)
+
+kern_status_t futex_init(void)
+{
+	vm_cache_init(&futex_cache);
+	return KERN_OK;
+}
+
+static kern_status_t get_data(
+	futex_key_t key,
+	unsigned int flags,
+	struct futex **out,
+	spin_lock_t **out_lock,
+	unsigned long *irq_flags)
+{
+	spin_lock_t *lock = NULL;
+	struct btree *futex_list = NULL;
+	struct task *self = NULL;
+
+	if (flags & FUTEX_PRIVATE) {
+		self = get_current_task();
+		lock = &self->t_base.ob_lock;
+		futex_list = &self->t_futex;
+	} else if (flags & FUTEX_SHARED) {
+		lock = &shared_futex_list_lock;
+		futex_list = &shared_futex_list;
+	} else {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	spin_lock_irqsave(lock, irq_flags);
+	struct futex *futex = get_futex(futex_list, key);
+
+	if (!futex && !(flags & FUTEX_CREATE)) {
+		spin_unlock_irqrestore(lock, *irq_flags);
+		if (self) {
+			put_current_task(self);
+		}
+
+		return KERN_NO_ENTRY;
+	}
+
+	futex = vm_cache_alloc(&futex_cache, VM_NORMAL);
+	if (!futex) {
+		spin_unlock_irqrestore(lock, *irq_flags);
+		if (self) {
+			put_current_task(self);
+		}
+
+		return KERN_NO_MEMORY;
+	}
+
+	futex->f_key = key;
+
+	put_futex(futex_list, futex);
+
+	if (self) {
+		put_current_task(self);
+	}
+
+	*out = futex;
+	*out_lock = lock;
+	return KERN_OK;
+}
+
+static kern_status_t cleanup_data(struct futex *futex, unsigned int flags)
+{
+	struct task *self = NULL;
+	struct btree *futex_list = NULL;
+	if (flags & FUTEX_PRIVATE) {
+		self = get_current_task();
+		futex_list = &self->t_futex;
+	} else if (flags & FUTEX_SHARED) {
+		futex_list = &shared_futex_list;
+	} else {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	btree_delete(futex_list, &futex->f_node);
+	vm_cache_free(&futex_cache, futex);
+
+	if (self) {
+		put_current_task(self);
+	}
+
+	return KERN_OK;
+}
+
+static kern_status_t futex_get_shared(kern_futex_t *futex, futex_key_t *out)
+{
+	struct task *self = get_current_task();
+	struct address_space *space = self->t_address_space;
+
+	unsigned long flags;
+	address_space_lock_irqsave(space, &flags);
+	kern_status_t status = address_space_translate(
+		space,
+		(virt_addr_t)futex,
+		out,
+		&flags);
+	address_space_unlock_irqrestore(space, flags);
+	put_current_task(self);
+	return status;
+}
+
+static kern_status_t futex_get_private(kern_futex_t *futex, futex_key_t *out)
+{
+	*out = (futex_key_t)futex;
+	return KERN_OK;
+}
+
+kern_status_t futex_get(
+	kern_futex_t *futex,
+	futex_key_t *out,
+	unsigned int flags)
+{
+	if (flags & FUTEX_PRIVATE) {
+		return futex_get_private(futex, out);
+	}
+
+	if (flags & FUTEX_SHARED) {
+		return futex_get_shared(futex, out);
+	}
+
+	return KERN_INVALID_ARGUMENT;
+}
+
+static kern_status_t futex_read(
+	struct futex *futex,
+	unsigned int flags,
+	kern_futex_t *out)
+{
+	if (flags & FUTEX_PRIVATE) {
+		virt_addr_t addr = futex->f_key;
+		*out = *(kern_futex_t *)addr;
+		return KERN_OK;
+	}
+
+	if (flags & FUTEX_SHARED) {
+		phys_addr_t paddr = futex->f_key;
+		virt_addr_t vaddr = (virt_addr_t)vm_phys_to_virt(paddr);
+		if (!vaddr) {
+			return KERN_MEMORY_FAULT;
+		}
+
+		*out = *(kern_futex_t *)vaddr;
+		return KERN_OK;
+	}
+
+	return KERN_INVALID_ARGUMENT;
+}
+
+kern_status_t futex_wait(
+	futex_key_t key,
+	kern_futex_t new_val,
+	unsigned int flags)
+{
+	spin_lock_t *lock = NULL;
+	unsigned long irq_flags = 0;
+	struct futex *futex = NULL;
+	kern_status_t status = get_data(key, flags, &futex, &lock, &irq_flags);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	kern_futex_t current_val = 0;
+	status = futex_read(futex, flags, &current_val);
+	if (status != KERN_OK) {
+		spin_unlock_irqrestore(lock, irq_flags);
+		return status;
+	}
+
+	if (current_val != new_val) {
+		spin_unlock_irqrestore(lock, irq_flags);
+		return KERN_BAD_STATE;
+	}
+
+	struct wait_item waiter;
+	thread_wait_begin(&waiter, &futex->f_waiters);
+	spin_unlock_irqrestore(lock, irq_flags);
+
+	schedule(SCHED_NORMAL);
+
+	spin_lock_irqsave(lock, &irq_flags);
+	thread_wait_end(&waiter, &futex->f_waiters);
+
+	if (waitqueue_empty(&futex->f_waiters)) {
+		cleanup_data(futex, flags);
+	}
+
+	spin_unlock_irqrestore(lock, irq_flags);
+
+	return KERN_OK;
+}
+
+kern_status_t futex_wake(futex_key_t key, size_t nwaiters, unsigned int flags)
+{
+	spin_lock_t *lock = NULL;
+	unsigned long irq_flags = 0;
+	struct futex *futex = NULL;
+	kern_status_t status = get_data(key, flags, &futex, &lock, &irq_flags);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	if (nwaiters == FUTEX_WAKE_ALL) {
+		wakeup_queue(&futex->f_waiters);
+	} else {
+		wakeup_n(&futex->f_waiters, nwaiters);
+	}
+
+	spin_unlock_irqrestore(lock, irq_flags);
+	return KERN_OK;
+}

kernel/handle.c

@@ -5,14 +5,14 @@
 #include <kernel/sched.h>
 #include <kernel/util.h>
 #include <kernel/vm.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 /* depth=3 gives a maximum of ~66.6 million handles */
 #define MAX_TABLE_DEPTH  3
 #define RESERVED_HANDLES 64
 
 static struct vm_cache handle_table_cache = {
-	.c_name = "handle_table",
+	.c_name = "handle-table",
 	.c_obj_size = sizeof(struct handle_table),
 };
 
@@ -33,8 +33,123 @@ struct handle_table *handle_table_create(void)
 	return out;
 }
 
+static void do_handle_table_destroy_leaf(struct handle_table *tab)
+{
+	while (1) {
+		unsigned int index = bitmap_lowest_set(
+			tab->t_handles.t_handle_map,
+			HANDLES_PER_TABLE);
+		if (index == BITMAP_NPOS) {
+			break;
+		}
+
+		struct handle *child = &tab->t_handles.t_handle_list[index];
+		bitmap_clear(tab->t_subtables.t_subtable_map, index);
+		if (child->h_object) {
+			object_unref(child->h_object);
+			child->h_object = NULL;
+		}
+	}
+}
+
+static void do_handle_table_destroy(
+	struct handle_table *tab,
+	unsigned int depth)
+{
+	if (depth == MAX_TABLE_DEPTH - 1) {
+		do_handle_table_destroy_leaf(tab);
+		return;
+	}
+
+	for (size_t i = 0; i < REFS_PER_TABLE; i++) {
+		struct handle_table *child
+			= tab->t_subtables.t_subtable_list[i];
+		if (child) {
+			do_handle_table_destroy(child, depth + 1);
+		}
+	}
+
+	vm_cache_free(&handle_table_cache, tab);
+}
+
 void handle_table_destroy(struct handle_table *tab)
 {
+	do_handle_table_destroy(tab, 0);
+}
+
+static kern_status_t do_handle_table_duplicate_leaf(
+	struct handle_table *src,
+	struct handle_table **dest)
+{
+	struct handle_table *out
+		= vm_cache_alloc(&handle_table_cache, VM_NORMAL);
+	if (!out) {
+		return KERN_NO_MEMORY;
+	}
+
+	memcpy(out, src, sizeof *out);
+
+	for (size_t i = 0; i < HANDLES_PER_TABLE; i++) {
+		struct object *obj = src->t_handles.t_handle_list[i].h_object;
+		if (obj) {
+			object_ref(obj);
+		}
+	}
+
+	*dest = out;
+	return KERN_OK;
+}
+
+static kern_status_t do_handle_table_duplicate(
+	struct handle_table *src,
+	struct handle_table **dest,
+	unsigned int depth)
+{
+	if (depth == MAX_TABLE_DEPTH - 1) {
+		return do_handle_table_duplicate_leaf(src, dest);
+	}
+
+	struct handle_table *out
+		= vm_cache_alloc(&handle_table_cache, VM_NORMAL);
+	if (!out) {
+		return KERN_NO_MEMORY;
+	}
+
+	memcpy(out->t_subtables.t_subtable_map,
+	       src->t_subtables.t_subtable_map,
+	       sizeof out->t_subtables.t_subtable_map);
+	memset(out->t_subtables.t_subtable_list,
+	       0x0,
+	       sizeof out->t_subtables.t_subtable_list);
+
+	for (size_t i = 0; i < REFS_PER_TABLE; i++) {
+		struct handle_table *child
+			= src->t_subtables.t_subtable_list[i];
+		struct handle_table *dup = NULL;
+		kern_status_t status = KERN_OK;
+		if (child) {
+			status = do_handle_table_duplicate(
+				child,
+				&dup,
+				depth + 1);
+		}
+
+		if (status == KERN_OK) {
+			out->t_subtables.t_subtable_list[i] = dup;
+		} else {
+			return status;
+		}
+	}
+
+	*dest = out;
+	return KERN_OK;
+}
+
+kern_status_t handle_table_duplicate(
+	struct handle_table *src,
+	struct handle_table **dest)
+{
+	return do_handle_table_duplicate(src, dest, 0);
 }
 
 static kern_status_t decode_handle_indices(
@@ -69,7 +184,7 @@ static kern_status_t encode_handle_indices(
 	return KERN_OK;
 }
 
-kern_status_t handle_table_alloc_handle(
+static kern_status_t alloc_handle(
 	struct handle_table *tab,
 	struct handle **out_slot,
 	kern_handle_t *out_handle)
@@ -124,6 +239,53 @@ kern_status_t handle_table_alloc_handle(
 	return encode_handle_indices(indices, out_handle);
 }
 
+kern_status_t handle_table_alloc_handle(
+	struct handle_table *tab,
+	kern_handle_t value,
+	struct handle **out_slot,
+	kern_handle_t *out_handle)
+{
+	if (value == KERN_HANDLE_INVALID) {
+		return alloc_handle(tab, out_slot, out_handle);
+	}
+
+	unsigned int indices[MAX_TABLE_DEPTH];
+	if (decode_handle_indices(value, indices) != KERN_OK) {
+		return KERN_HANDLE_INVALID;
+	}
+
+	int i;
+	for (i = 0; i < MAX_TABLE_DEPTH - 1; i++) {
+		struct handle_table *next
+			= tab->t_subtables.t_subtable_list[indices[i]];
+		if (!next) {
+			next = handle_table_create();
+			tab->t_subtables.t_subtable_list[indices[i]] = next;
+		}
+
+		if (!next) {
+			return KERN_NO_MEMORY;
+		}
+
+		tab = next;
+	}
+
+	unsigned int handle_index = indices[i];
+	bitmap_set(tab->t_handles.t_handle_map, handle_index);
+
+	struct handle *out = &tab->t_handles.t_handle_list[handle_index];
+
+	if (out->h_object) {
+		object_unref(out->h_object);
+		out->h_object = NULL;
+		out->h_flags = 0;
+	}
+
+	*out_slot = out;
+	*out_handle = value;
+	return KERN_OK;
+}
+
 kern_status_t handle_table_free_handle(
 	struct handle_table *tab,
 	kern_handle_t handle)
@@ -155,7 +317,7 @@ kern_status_t handle_table_free_handle(
 		= &tab->t_handles.t_handle_list[handle_index];
 
 	if (handle_entry->h_object) {
-		object_remove_handle(handle_entry->h_object);
+		object_unref(handle_entry->h_object);
 	}
 
 	memset(handle_entry, 0x0, sizeof *handle_entry);
@@ -254,11 +416,12 @@ kern_status_t handle_table_transfer(
 		}
 
 		switch (src_handle.hnd_mode) {
-		case KERN_MSG_HANDLE_IGNORE:
+		case HANDLE_TRANSFER_IGNORE:
 			break;
-		case KERN_MSG_HANDLE_MOVE:
+		case HANDLE_TRANSFER_MOVE:
 			status = handle_table_alloc_handle(
 				dst,
+				KERN_HANDLE_INVALID,
 				&dst_entry,
 				&dst_value);
 			if (status != KERN_OK) {
@@ -267,7 +430,7 @@ kern_status_t handle_table_transfer(
 
 			dst_entry->h_object = src_entry->h_object;
 			dst_entry->h_flags = src_entry->h_flags;
-			object_add_handle(dst_entry->h_object);
+			object_ref(dst_entry->h_object);
 
 			handle_table_free_handle(src, src_handles[i].hnd_value);
 
@@ -275,9 +438,10 @@ kern_status_t handle_table_transfer(
 			dst_handle.hnd_value = dst_value;
 			dst_handle.hnd_result = KERN_OK;
 			break;
-		case KERN_MSG_HANDLE_COPY:
+		case HANDLE_TRANSFER_COPY:
 			status = handle_table_alloc_handle(
 				dst,
+				KERN_HANDLE_INVALID,
 				&dst_entry,
 				&dst_value);
 			if (status != KERN_OK) {
@@ -286,7 +450,7 @@ kern_status_t handle_table_transfer(
 
 			dst_entry->h_object = src_entry->h_object;
 			dst_entry->h_flags = src_entry->h_flags;
-			object_add_handle(dst_entry->h_object);
+			object_ref(dst_entry->h_object);
 
 			dst_handle.hnd_mode = src_handles[i].hnd_mode;
 			dst_handle.hnd_value = dst_value;
@@ -324,14 +488,14 @@ kern_status_t handle_table_transfer(
 			&handle,
 			NULL);
 
-		if (handle.hnd_mode != KERN_MSG_HANDLE_MOVE) {
+		if (handle.hnd_mode != HANDLE_TRANSFER_MOVE) {
 			continue;
 		}
 
 		struct handle *src_entry
 			= handle_table_get_handle(src, handle.hnd_value);
 		if (src_entry) {
-			object_remove_handle(src_entry->h_object);
+			object_unref(src_entry->h_object);
 			handle_table_free_handle(src, handle.hnd_value);
 		}
 	}

kernel/msg.c

@@ -0,0 +1,22 @@
+#include <kernel/msg.h>
+#include <kernel/vm.h>
+
+static struct vm_cache msg_cache = {
+	.c_name = "msg",
+	.c_obj_size = sizeof(struct msg),
+};
+
+void msg_init(void)
+{
+	vm_cache_init(&msg_cache);
+}
+
+struct msg *msg_alloc(void)
+{
+	return vm_cache_alloc(&msg_cache, VM_NORMAL);
+}
+
+void msg_free(struct msg *msg)
+{
+	vm_cache_free(&msg_cache, msg);
+}

kernel/object.c

@@ -74,90 +74,28 @@ struct object *object_create(struct object_type *type)
 	obj->ob_lock = SPIN_LOCK_INIT;
 	obj->ob_magic = OBJECT_MAGIC;
 	obj->ob_refcount = 1;
-	obj->ob_handles = 0;
 
 	return obj;
 }
 
 struct object *object_ref(struct object *obj)
 {
-	obj->ob_refcount++;
+	atomic_add_fetch(&obj->ob_refcount, 1);
 	return obj;
 }
 
-static void __cleanup(struct object *obj, struct queue *queue)
-{
-	if (HAS_OP(obj, destroy)) {
-		obj->ob_type->ob_ops.destroy(obj, queue);
-	}
-
-	vm_cache_free(&obj->ob_type->ob_cache, obj);
-}
-
-static void object_cleanup(struct object *obj, unsigned long flags)
-{
-	if (obj->ob_refcount > 0 || obj->ob_handles > 0) {
-		spin_unlock_irqrestore(&obj->ob_lock, flags);
-		return;
-	}
-
-	struct queue queue = QUEUE_INIT;
-	__cleanup(obj, &queue);
-
-	if (!HAS_OP(obj, destroy_recurse)) {
-		return;
-	}
-
-	while (!queue_empty(&queue)) {
-		struct queue_entry *entry = queue_pop_front(&queue);
-		struct object *child = NULL;
-		obj->ob_type->ob_ops.destroy_recurse(entry, &child);
-		if (!child) {
-			continue;
-		}
-
-		if (child->ob_refcount > 1) {
-			child->ob_refcount--;
-			continue;
-		}
-
-		if (child->ob_refcount == 0 && child->ob_handles == 0) {
-			__cleanup(child, &queue);
-		}
-	}
-}
-
 void object_unref(struct object *obj)
 {
-	unsigned long flags;
-	spin_lock_irqsave(&obj->ob_lock, &flags);
-
-	if (obj->ob_refcount == 0) {
-		spin_unlock_irqrestore(&obj->ob_lock, flags);
+	int ref = atomic_sub_fetch(&obj->ob_refcount, 1);
+	if (ref > 0) {
 		return;
 	}
 
-	obj->ob_refcount--;
-	object_cleanup(obj, flags);
+	if (HAS_OP(obj, destroy)) {
+		obj->ob_type->ob_ops.destroy(obj);
 	}
 
-void object_add_handle(struct object *obj)
-{
-	obj->ob_handles++;
-}
-
-void object_remove_handle(struct object *obj)
-{
-	unsigned long flags;
-	spin_lock_irqsave(&obj->ob_lock, &flags);
-
-	if (obj->ob_handles == 0) {
-		spin_unlock_irqrestore(&obj->ob_lock, flags);
-		return;
-	}
-
-	obj->ob_handles--;
-	object_cleanup(obj, flags);
+	vm_cache_free(&obj->ob_type->ob_cache, obj);
 }
 
 void object_lock(struct object *obj)
@@ -237,7 +175,7 @@ void object_wait_signal(
 	uint32_t signals,
 	unsigned long *irq_flags)
 {
-	struct thread *self = current_thread();
+	struct thread *self = get_current_thread();
 	struct wait_item waiter;
 	wait_item_init(&waiter, self);
 	for (;;) {
@@ -251,4 +189,5 @@ void object_wait_signal(
 		object_lock_irqsave(obj, irq_flags);
 	}
 	thread_wait_end(&waiter, &obj->ob_wq);
+	put_current_thread(self);
 }

kernel/panic.c

@@ -19,8 +19,8 @@ void panic_irq(struct ml_cpu_context *ctx, const char *fmt, ...)
 	printk("---[ kernel panic: %s", buf);
 	printk("kernel: " BUILD_ID ", compiler version: " __VERSION__);
 
-	struct task *task = current_task();
-	struct thread *thr = current_thread();
+	struct task *task = get_current_task();
+	struct thread *thr = get_current_thread();
 
 	if (task && thr) {
 		printk("task: %s (id: %d, thread: %d)",
@@ -31,6 +31,9 @@ void panic_irq(struct ml_cpu_context *ctx, const char *fmt, ...)
 		printk("task: [bootstrap]");
 	}
 
+	put_current_thread(thr);
+	put_current_task(task);
+
 	printk("cpu: %u", this_cpu());
 
 	ml_print_cpu_state(ctx);

kernel/port.c

@@ -1,16 +1,29 @@
 #include <kernel/channel.h>
 #include <kernel/port.h>
+#include <kernel/printk.h>
 #include <kernel/thread.h>
 #include <kernel/util.h>
 
 #define PORT_CAST(p) OBJECT_C_CAST(struct port, p_base, &port_type, p)
 
+static kern_status_t port_cleanup(struct object *obj);
+
 static struct object_type port_type = {
 	.ob_name = "port",
 	.ob_size = sizeof(struct port),
 	.ob_header_offset = offsetof(struct port, p_base),
+	.ob_ops = {
+		.destroy = port_cleanup,
+	},
 };
 
+static kern_status_t port_cleanup(struct object *obj)
+{
+	struct port *port = PORT_CAST(obj);
+	port_disconnect(port);
+	return KERN_OK;
+}
+
 kern_status_t port_type_init(void)
 {
 	return object_type_register(&port_type);
@@ -24,7 +37,7 @@ struct port *port_cast(struct object *obj)
 static void wait_for_reply(struct msg *msg, unsigned long *lock_flags)
 {
 	struct wait_item waiter;
-	struct thread *self = current_thread();
+	struct thread *self = get_current_thread();
 
 	wait_item_init(&waiter, self);
 	for (;;) {
@@ -39,6 +52,7 @@ static void wait_for_reply(struct msg *msg, unsigned long *lock_flags)
 	}
 
 	self->tr_state = THREAD_READY;
+	put_current_thread(self);
 }
 
 struct port *port_create(void)
@@ -58,9 +72,29 @@ struct port *port_create(void)
 kern_status_t port_connect(struct port *port, struct channel *remote)
 {
 	if (port->p_status != PORT_OFFLINE) {
+		tracek("port_connect: port in bad state (%d)", port->p_status);
 		return KERN_BAD_STATE;
 	}
 
+	struct msg *msg = msg_alloc();
+	if (!msg) {
+		return KERN_NO_MEMORY;
+	}
+
+	msg->msg_status = KMSG_ASYNC;
+	msg->msg_type = KERN_MSG_TYPE_EVENT;
+	msg->msg_event = KERN_MSG_EVENT_CONNECTION;
+	msg->msg_sender_port_id = port->p_base.ob_id;
+
+	struct thread *self = get_current_thread();
+	msg->msg_sender_thread_id = self->tr_id;
+	put_current_thread(self);
+
+	unsigned long flags;
+	channel_lock_irqsave(remote, &flags);
+	channel_enqueue_msg(remote, msg);
+	channel_unlock_irqrestore(remote, flags);
+
 	port->p_remote = remote;
 	port->p_status = PORT_READY;
 	return KERN_OK;
@@ -69,9 +103,30 @@ kern_status_t port_connect(struct port *port, struct channel *remote)
 kern_status_t port_disconnect(struct port *port)
 {
 	if (port->p_status != PORT_READY) {
+		tracek("port_disconnect: port in bad state (%d)",
+		       port->p_status);
 		return KERN_BAD_STATE;
 	}
 
+	struct msg *msg = msg_alloc();
+	if (!msg) {
+		return KERN_NO_MEMORY;
+	}
+
+	msg->msg_status = KMSG_ASYNC;
+	msg->msg_type = KERN_MSG_TYPE_EVENT;
+	msg->msg_event = KERN_MSG_EVENT_DISCONNECTION;
+	msg->msg_sender_port_id = port->p_base.ob_id;
+
+	struct thread *self = get_current_thread();
+	msg->msg_sender_thread_id = self->tr_id;
+	put_current_thread(self);
+
+	unsigned long flags;
+	channel_lock_irqsave(port->p_remote, &flags);
+	channel_enqueue_msg(port->p_remote, msg);
+	channel_unlock_irqrestore(port->p_remote, flags);
+
 	port->p_remote = NULL;
 	port->p_status = PORT_OFFLINE;
 	return KERN_OK;
@@ -84,29 +139,31 @@ kern_status_t port_send_msg(
 	unsigned long *lock_flags)
 {
 	if (port->p_status != PORT_READY) {
+		tracek("port_send_msg: port in bad state (%d)", port->p_status);
 		return KERN_BAD_STATE;
 	}
 
-	struct thread *self = current_thread();
-	struct msg *msg = &self->tr_msg;
-	memset(msg, 0x0, sizeof *msg);
-	msg->msg_status = KMSG_WAIT_RECEIVE;
-	msg->msg_sender_thread = self;
-	msg->msg_sender_port = port;
-	memcpy(&msg->msg_req, in_msg, sizeof msg->msg_req);
-	memcpy(&msg->msg_resp, out_reply, sizeof msg->msg_req);
+	struct thread *self = get_current_thread();
+	struct msg msg;
+	memset(&msg, 0x0, sizeof msg);
+	msg.msg_type = KERN_MSG_TYPE_DATA;
+	msg.msg_status = KMSG_WAIT_RECEIVE;
+	msg.msg_sender_thread = self;
+	msg.msg_sender_port = port;
+	memcpy(&msg.msg_req, in_msg, sizeof msg.msg_req);
+	memcpy(&msg.msg_resp, out_reply, sizeof msg.msg_req);
 
 	unsigned long flags;
 	channel_lock_irqsave(port->p_remote, &flags);
-	port->p_status = PORT_SEND_BLOCKED;
-	channel_enqueue_msg(port->p_remote, msg);
+	channel_enqueue_msg(port->p_remote, &msg);
 	channel_unlock_irqrestore(port->p_remote, flags);
 
-	wait_for_reply(msg, lock_flags);
+	wait_for_reply(&msg, lock_flags);
 
 	channel_lock_irqsave(port->p_remote, &flags);
-	btree_delete(&port->p_remote->c_msg, &msg->msg_node);
+	btree_delete(&port->p_remote->c_msg, &msg.msg_node);
 	channel_unlock_irqrestore(port->p_remote, flags);
+	put_current_thread(self);
 
-	return msg->msg_result;
+	return msg.msg_result;
 }

kernel/printk.c

@@ -98,9 +98,8 @@ int printk(const char *format, ...)
 	unsigned long flags;
 	spin_lock_irqsave(&log_buffer_lock, &flags);
 	save_log_message(msg);
-	spin_unlock_irqrestore(&log_buffer_lock, flags);
-
 	flush_log_buffer();
+	spin_unlock_irqrestore(&log_buffer_lock, flags);
 
 	return 0;
 }

kernel/status.c

@@ -1,5 +1,5 @@
-#include <mango/status.h>
-#include <mango/types.h>
+#include <magenta/status.h>
+#include <magenta/types.h>
 
 #define ERROR_STRING_CASE(code)                                                \
 	case code:                                                             \

libc/string/memcmp.c

@@ -3,6 +3,7 @@
 int memcmp(const void *vl, const void *vr, size_t n)
 {
 	const unsigned char *l = vl, *r = vr;
-    for (; n && *l == *r; n--, l++, r++);
+	for (; n && *l == *r; n--, l++, r++)
+		;
 	return n ? *l - *r : 0;
 }

libc/string/memmove.c

@@ -1,5 +1,5 @@
-#include <stdint.h>
 #include <kernel/libc/string.h>
+#include <stdint.h>
 
 static void *memcpy_r(void *dest, const void *src, size_t sz)
 {

libmagenta/CMakeLists.txt

@@ -1,13 +1,12 @@
-file(GLOB headers ${CMAKE_CURRENT_SOURCE_DIR}/include/mango/*.h)
+file(GLOB headers ${CMAKE_CURRENT_SOURCE_DIR}/include/magenta/*.h)
 file(GLOB asm_sources
 	${CMAKE_CURRENT_SOURCE_DIR}/arch/${CMAKE_SYSTEM_PROCESSOR}/*.S)
-set_property(SOURCE ${asm_sources} PROPERTY LANGUAGE C)
 
 set(public_include_dirs
 	${CMAKE_CURRENT_SOURCE_DIR}/include
 	${CMAKE_CURRENT_SOURCE_DIR}/include-user)
 
-add_library(libmango STATIC ${asm_sources})
-target_include_directories(libmango PUBLIC
+add_library(libmagenta STATIC ${asm_sources})
+target_include_directories(libmagenta PUBLIC
 	${CMAKE_CURRENT_SOURCE_DIR}/include
 	${CMAKE_CURRENT_SOURCE_DIR}/include-user)

libmagenta/arch/x86_64/syscall.S

@@ -1,4 +1,4 @@
-#include "mango/syscall.h"
+#include "magenta/syscall.h"
 
 # Registers:
 #  rax = syscall ID + return value
@@ -56,12 +56,19 @@
 .endm
 
 SYSCALL_GATE task_exit SYS_TASK_EXIT 1
-SYSCALL_GATE task_self SYS_TASK_SELF 0
-SYSCALL_GATE task_create SYS_TASK_CREATE 5
+SYSCALL_GATE task_self SYS_TASK_SELF 1
+SYSCALL_GATE task_create SYS_TASK_CREATE 6
+SYSCALL_GATE task_duplicate SYS_TASK_DUPLICATE 2
 SYSCALL_GATE task_create_thread SYS_TASK_CREATE_THREAD 6
 SYSCALL_GATE task_get_address_space SYS_TASK_GET_ADDRESS_SPACE 1
+SYSCALL_GATE task_config_get SYS_TASK_CONFIG_GET 4
+SYSCALL_GATE task_config_set SYS_TASK_CONFIG_SET 4
 
+SYSCALL_GATE thread_self SYS_THREAD_SELF 1
 SYSCALL_GATE thread_start SYS_THREAD_START 1
+SYSCALL_GATE thread_exit SYS_THREAD_EXIT 0
+SYSCALL_GATE thread_config_get SYS_THREAD_CONFIG_GET 4
+SYSCALL_GATE thread_config_set SYS_THREAD_CONFIG_SET 4
 
 SYSCALL_GATE vm_object_create SYS_VM_OBJECT_CREATE 5
 SYSCALL_GATE vm_object_read SYS_VM_OBJECT_READ 5
@@ -70,14 +77,14 @@ SYSCALL_GATE vm_object_copy SYS_VM_OBJECT_COPY 6
 
 SYSCALL_GATE address_space_read SYS_ADDRESS_SPACE_READ 5
 SYSCALL_GATE address_space_write SYS_ADDRESS_SPACE_WRITE 5
-SYSCALL_GATE address_space_map SYS_ADDRESS_SPACE_MAP 7
+SYSCALL_GATE address_space_map SYS_ADDRESS_SPACE_MAP 8
 SYSCALL_GATE address_space_unmap SYS_ADDRESS_SPACE_UNMAP 3
 SYSCALL_GATE address_space_reserve SYS_ADDRESS_SPACE_RESERVE 4
 SYSCALL_GATE address_space_release SYS_ADDRESS_SPACE_RELEASE 3
 
 SYSCALL_GATE kern_log SYS_KERN_LOG 1
 SYSCALL_GATE kern_handle_close SYS_KERN_HANDLE_CLOSE 1
-SYSCALL_GATE kern_handle_duplicate SYS_KERN_HANDLE_DUPLICATE 2
+SYSCALL_GATE kern_handle_transfer SYS_KERN_HANDLE_TRANSFER 6
 SYSCALL_GATE kern_config_get SYS_KERN_CONFIG_GET 3
 SYSCALL_GATE kern_config_set SYS_KERN_CONFIG_SET 3
 
@@ -95,8 +102,14 @@ SYSCALL_GATE vm_controller_create SYS_VM_CONTROLLER_CREATE 1
 SYSCALL_GATE vm_controller_recv SYS_VM_CONTROLLER_RECV 2
 SYSCALL_GATE vm_controller_recv_async SYS_VM_CONTROLLER_RECV_ASYNC 3
 SYSCALL_GATE vm_controller_create_object SYS_VM_CONTROLLER_CREATE_OBJECT 7
+SYSCALL_GATE vm_controller_prepare_attach SYS_VM_CONTROLLER_PREPARE_ATTACH 3
+SYSCALL_GATE vm_controller_finish_attach SYS_VM_CONTROLLER_FINISH_ATTACH 3
 SYSCALL_GATE vm_controller_detach_object SYS_VM_CONTROLLER_DETACH_OBJECT 2
 SYSCALL_GATE vm_controller_supply_pages SYS_VM_CONTROLLER_SUPPLY_PAGES 6
 
 SYSCALL_GATE kern_object_wait SYS_KERN_OBJECT_WAIT 2
+SYSCALL_GATE kern_object_query SYS_KERN_OBJECT_QUERY 2
+
+SYSCALL_GATE futex_wait SYS_FUTEX_WAIT 3
+SYSCALL_GATE futex_wake SYS_FUTEX_WAKE 3
 

libmagenta/include-user/magenta/config.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_CONFIG_H_
-#define MANGO_CONFIG_H_
+#ifndef MAGENTA_CONFIG_H_
+#define MAGENTA_CONFIG_H_
 
-#include <mango/status.h>
-#include <mango/types.h>
+#include <magenta/status.h>
+#include <magenta/types.h>
 #include <stddef.h>
 
 extern kern_status_t kern_config_get(

libmagenta/include-user/magenta/equeue.h

@@ -1,7 +1,7 @@
-#ifndef MANGO_EQUEUE_H_
-#define MANGO_EQUEUE_H_
+#ifndef MAGENTA_EQUEUE_H_
+#define MAGENTA_EQUEUE_H_
 
-#include <mango/types.h>
+#include <magenta/types.h>
 
 extern kern_status_t equeue_create(kern_handle_t *out);
 extern kern_status_t equeue_dequeue(kern_handle_t eq, equeue_packet_t *out);

libmagenta/include-user/magenta/futex.h

@@ -0,0 +1,16 @@
+#ifndef MAGENTA_FUTEX_H_
+#define MAGENTA_FUTEX_H_
+
+#include <magenta/status.h>
+#include <magenta/types.h>
+
+extern kern_status_t futex_wait(
+	kern_futex_t *futex,
+	kern_futex_t new_val,
+	unsigned int flags);
+extern kern_status_t futex_wake(
+	kern_futex_t *futex,
+	unsigned int nr_waiters,
+	unsigned int flags);
+
+#endif

libmagenta/include-user/magenta/handle.h

@@ -0,0 +1,22 @@
+#ifndef MAGENTA_HANDLE_H_
+#define MAGENTA_HANDLE_H_
+
+#include <magenta/status.h>
+#include <magenta/types.h>
+
+extern kern_status_t kern_handle_close(kern_handle_t handle);
+extern kern_status_t kern_handle_transfer(
+	kern_handle_t src_task,
+	kern_handle_t src_handle,
+	kern_handle_t dest_task,
+	kern_handle_t dest_handle,
+	unsigned int mode,
+	kern_handle_t *out_handle);
+extern kern_status_t kern_handle_control(
+	kern_handle_t task,
+	kern_handle_t handle,
+	uint32_t set_mask,
+	uint32_t clear_mask,
+	uint32_t *out_flags);
+
+#endif

libmagenta/include-user/magenta/log.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_LOG_H_
-#define MANGO_LOG_H_
+#ifndef MAGENTA_LOG_H_
+#define MAGENTA_LOG_H_
 
-#include <mango/status.h>
-#include <mango/types.h>
+#include <magenta/status.h>
+#include <magenta/types.h>
 
 #undef TRACE
 

libmagenta/include-user/magenta/msg.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_MSG_H_
-#define MANGO_MSG_H_
+#ifndef MAGENTA_MSG_H_
+#define MAGENTA_MSG_H_
 
-#include <mango/status.h>
-#include <mango/types.h>
+#include <magenta/status.h>
+#include <magenta/types.h>
 
 extern kern_status_t channel_create(unsigned int id, kern_handle_t *out);
 extern kern_status_t port_create(kern_handle_t *out);

libmagenta/include-user/magenta/object.h

@@ -0,0 +1,11 @@
+#ifndef MAGENTA_OBJECT_H_
+#define MAGENTA_OBJECT_H_
+
+#include <magenta/types.h>
+
+extern kern_status_t kern_object_wait(kern_wait_item_t *items, size_t nr_items);
+extern kern_status_t kern_object_query(
+	kern_handle_t handle,
+	kern_object_info_t *out);
+
+#endif

libmagenta/include-user/magenta/task.h

@@ -0,0 +1,62 @@
+#ifndef MAGENTA_TASK_H_
+#define MAGENTA_TASK_H_
+
+#include <magenta/status.h>
+#include <magenta/types.h>
+
+extern kern_status_t task_exit(int status);
+extern kern_status_t task_self(kern_handle_t *out);
+
+extern kern_status_t task_create(
+	kern_handle_t parent,
+	task_flags_t flags,
+	const char *name,
+	size_t name_len,
+	kern_handle_t *out_task,
+	kern_handle_t *out_address_space);
+extern kern_status_t task_create_thread(
+	kern_handle_t task,
+	virt_addr_t ip,
+	virt_addr_t sp,
+	uintptr_t *args,
+	size_t nr_args,
+	kern_handle_t *out_thread);
+extern kern_status_t task_get_address_space(
+	kern_handle_t task,
+	kern_handle_t *out);
+extern kern_status_t task_config_get(
+	kern_handle_t task,
+	kern_config_key_t key,
+	void *ptr,
+	size_t len);
+extern kern_status_t task_config_set(
+	kern_handle_t task,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+extern kern_status_t task_duplicate(
+	kern_handle_t *out_task,
+	kern_handle_t *out_address_space);
+extern kern_status_t task_reset(
+	virt_addr_t ip,
+	virt_addr_t sp,
+	uintptr_t *args,
+	size_t nr_args,
+	virt_addr_t unmap_base,
+	size_t unmap_length);
+
+extern kern_status_t thread_self(kern_handle_t *out);
+extern kern_status_t thread_start(kern_handle_t thread);
+extern kern_status_t thread_exit(void);
+extern kern_status_t thread_config_get(
+	kern_handle_t thread,
+	kern_config_key_t key,
+	void *ptr,
+	size_t len);
+extern kern_status_t thread_config_set(
+	kern_handle_t thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+
+#endif

libmagenta/include-user/magenta/vm.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_VM_H_
-#define MANGO_VM_H_
+#ifndef MAGENTA_VM_H_
+#define MAGENTA_VM_H_
 
-#include <mango/status.h>
-#include <mango/types.h>
+#include <magenta/status.h>
+#include <magenta/types.h>
 
 extern kern_status_t vm_object_create(
 	const char *name,
@@ -48,6 +48,7 @@ extern kern_status_t address_space_map(
 	kern_handle_t object,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out_base_address);
 extern kern_status_t address_space_unmap(
@@ -67,7 +68,7 @@ extern kern_status_t address_space_release(
 extern kern_status_t vm_controller_create(kern_handle_t *out);
 extern kern_status_t vm_controller_recv(
 	kern_handle_t ctrl,
-	equeue_packet_page_request_t *out);
+	equeue_packet_vm_request_t *out);
 extern kern_status_t vm_controller_recv_async(
 	kern_handle_t ctrl,
 	kern_handle_t eq,
@@ -80,6 +81,14 @@ extern kern_status_t vm_controller_create_object(
 	size_t data_len,
 	vm_prot_t prot,
 	kern_handle_t *out);
+extern kern_status_t vm_controller_prepare_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	kern_handle_t *out_vmo);
+extern kern_status_t vm_controller_finish_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key);
 extern kern_status_t vm_controller_detach_object(
 	kern_handle_t ctrl,
 	kern_handle_t vmo);

libmagenta/include/magenta/signal.h

@@ -1,5 +1,7 @@
-#ifndef MANGO_SIGNAL_H_
-#define MANGO_SIGNAL_H_
+#ifndef MAGENTA_SIGNAL_H_
+#define MAGENTA_SIGNAL_H_
+
+#define THREAD_SIGNAL_STOPPED                 0x01u
 
 #define CHANNEL_SIGNAL_MSG_RECEIVED           0x01u
 

libmagenta/include/magenta/status.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_STATUS_H_
-#define MANGO_STATUS_H_
+#ifndef MAGENTA_STATUS_H_
+#define MAGENTA_STATUS_H_
 
 #define KERN_OK               (0)
 #define KERN_UNIMPLEMENTED    (1)

libmagenta/include/magenta/syscall.h

@@ -0,0 +1,58 @@
+#ifndef MAGENTA_SYSCALL_H_
+#define MAGENTA_SYSCALL_H_
+
+#define SYS_KERN_LOG                     1
+#define SYS_KERN_HANDLE_CLOSE            2
+#define SYS_KERN_HANDLE_TRANSFER         3
+#define SYS_KERN_HANDLE_CONTROL          4
+#define SYS_KERN_CONFIG_GET              5
+#define SYS_KERN_CONFIG_SET              6
+#define SYS_KERN_OBJECT_WAIT             7
+#define SYS_KERN_OBJECT_WAIT_ASYNC       8
+#define SYS_TASK_EXIT                    9
+#define SYS_TASK_SELF                    10
+#define SYS_TASK_CREATE                  11
+#define SYS_TASK_CREATE_THREAD           12
+#define SYS_TASK_GET_ADDRESS_SPACE       13
+#define SYS_TASK_CONFIG_GET              14
+#define SYS_TASK_CONFIG_SET              15
+#define SYS_TASK_DUPLICATE               16
+#define SYS_THREAD_SELF                  17
+#define SYS_THREAD_START                 18
+#define SYS_THREAD_EXIT                  19
+#define SYS_THREAD_CONFIG_GET            20
+#define SYS_THREAD_CONFIG_SET            21
+#define SYS_VM_OBJECT_CREATE             22
+#define SYS_VM_OBJECT_READ               23
+#define SYS_VM_OBJECT_WRITE              24
+#define SYS_VM_OBJECT_COPY               25
+#define SYS_ADDRESS_SPACE_READ           26
+#define SYS_ADDRESS_SPACE_WRITE          27
+#define SYS_ADDRESS_SPACE_MAP            28
+#define SYS_ADDRESS_SPACE_UNMAP          29
+#define SYS_ADDRESS_SPACE_RESERVE        30
+#define SYS_ADDRESS_SPACE_RELEASE        31
+#define SYS_MSG_SEND                     32
+#define SYS_MSG_RECV                     33
+#define SYS_MSG_REPLY                    34
+#define SYS_MSG_READ                     35
+#define SYS_MSG_WRITE                    36
+#define SYS_CHANNEL_CREATE               37
+#define SYS_PORT_CREATE                  38
+#define SYS_PORT_CONNECT                 39
+#define SYS_PORT_DISCONNECT              40
+#define SYS_EQUEUE_CREATE                41
+#define SYS_EQUEUE_DEQUEUE               42
+#define SYS_VM_CONTROLLER_CREATE         43
+#define SYS_VM_CONTROLLER_RECV           44
+#define SYS_VM_CONTROLLER_RECV_ASYNC     45
+#define SYS_VM_CONTROLLER_CREATE_OBJECT  46
+#define SYS_VM_CONTROLLER_PREPARE_ATTACH 47
+#define SYS_VM_CONTROLLER_FINISH_ATTACH  48
+#define SYS_VM_CONTROLLER_DETACH_OBJECT  49
+#define SYS_VM_CONTROLLER_SUPPLY_PAGES   50
+#define SYS_FUTEX_WAIT                   51
+#define SYS_FUTEX_WAKE                   52
+#define SYS_KERN_OBJECT_QUERY            53
+
+#endif

libmagenta/include/magenta/types.h

@@ -0,0 +1,239 @@
+#ifndef MAGENTA_TYPES_H_
+#define MAGENTA_TYPES_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#define VM_PROT_READ                 0x01u
+#define VM_PROT_WRITE                0x02u
+#define VM_PROT_EXEC                 0x04u
+#define VM_PROT_USER                 0x08u
+#define VM_PROT_SVR                  0x10u
+#define VM_PROT_NOCACHE              0x10u
+#define VM_PROT_MAP_SPECIFIC         0x40u
+
+#define MAP_ADDRESS_ANY              ((virt_addr_t) - 1)
+#define MAP_ADDRESS_INVALID          ((virt_addr_t)0)
+#define KERN_HANDLE_INVALID          ((kern_handle_t)0xFFFFFFFF)
+
+/* task creation flags */
+#define TASK_F_DEFAULT               0x0000u
+#define TASK_F_CLONE_ALL_HANDLES     0x0001u
+
+/* config keys for use with kern_config_get/kern_config_set */
+#define KERN_CFG_INVALID             0x00000u
+#define KERN_CFG_PAGE_SIZE           0x00001u
+
+/* config keys for use with task_config_get/task_config_set */
+#define TASK_CFG_INVALID             0x00000u
+#define TASK_CFG_ID                  0x10001u
+
+/* config keys for use with thread_config_get/thread_config_set */
+#define THREAD_CFG_INVALID           0x00000u
+#define THREAD_CFG_FSBASE            0x20001u
+#define THREAD_CFG_GSBASE            0x20002u
+
+/* user-defined flags that can be set on handles */
+#define KERN_HANDLE_FLAG0            0x10000000UL
+#define KERN_HANDLE_FLAG1            0x20000000UL
+#define KERN_HANDLE_FLAG2            0x40000000UL
+#define KERN_HANDLE_FLAG3            0x80000000UL
+
+/* flags to specify when creating address-space mappings */
+/* this mapping is private. if a task with this mapping is duplicated,
+   the duplicate task will receive a copy-on-write mapping. changes to one
+   mapping will not be visible to the other. */
+#define VM_PRIVATE                   0x0000u
+/* this mapping is shared. if a task with this mapping is duplicated,
+ * the duplicate will receive a mapping of the same data. changes to one mapping
+ * will be visibile to the other */
+#define VM_SHARED                    0x0001u
+
+/* maximum number of handles that can be sent in a single message */
+#define KERN_MSG_MAX_HANDLES         64
+
+/* the corresponding handle should be ignored */
+#define HANDLE_TRANSFER_IGNORE       0
+/* the corresponding handle should be moved to the recipient task. the handle
+ * will be closed. */
+#define HANDLE_TRANSFER_MOVE         1
+/* the corresponding handle should be copied to the recipient task. the handle
+ * will remain valid for the sending task. */
+#define HANDLE_TRANSFER_COPY         2
+
+/* maximum number of objects that can be waited on in a single call to
+ * kern_object_wait */
+#define KERN_WAIT_MAX_ITEMS          64
+
+/* message types */
+#define KERN_MSG_TYPE_NONE           0
+#define KERN_MSG_TYPE_DATA           1
+#define KERN_MSG_TYPE_EVENT          2
+
+/* event message types */
+#define KERN_MSG_EVENT_NONE          0
+#define KERN_MSG_EVENT_CONNECTION    1
+#define KERN_MSG_EVENT_DISCONNECTION 2
+
+/* equeue packet types */
+#define EQUEUE_PKT_VM_REQUEST        0x01u
+#define EQUEUE_PKT_ASYNC_SIGNAL      0x02u
+
+/* vm request types */
+#define VM_REQUEST_READ              0x01u
+#define VM_REQUEST_DIRTY             0x02u
+#define VM_REQUEST_ATTACH            0x03u
+#define VM_REQUEST_DETACH            0x04u
+
+/* futex special values */
+#define FUTEX_WAKE_ALL               ((size_t)-1)
+
+/* futex flags */
+#define FUTEX_PRIVATE                0x01u
+#define FUTEX_SHARED                 0x02u
+
+#define IOVEC(p, len)                                                          \
+	{                                                                      \
+		.io_base = (virt_addr_t)(p),                                   \
+		.io_len = (len),                                               \
+	}
+#define MSG_HANDLE(mode, value)                                                \
+	{                                                                      \
+		.hnd_mode = (mode),                                            \
+		.hnd_value = (value),                                          \
+	}
+#define MSG(data, data_count, handles, handles_len)                            \
+	{                                                                      \
+		.msg_data = (data),                                            \
+		.msg_data_count = (data_count),                                \
+		.msg_handles = (handles),                                      \
+		.msg_handles_count = (handles_len),                            \
+	}
+
+typedef uintptr_t phys_addr_t;
+typedef uintptr_t virt_addr_t;
+typedef uint64_t msgid_t;
+typedef uint64_t off_t;
+typedef uint64_t koid_t;
+typedef uintptr_t equeue_key_t;
+typedef unsigned int tid_t;
+typedef unsigned int vm_controller_packet_type_t;
+typedef unsigned int kern_status_t;
+typedef uint32_t kern_handle_t;
+typedef uint32_t kern_config_key_t;
+typedef uint32_t vm_prot_t;
+typedef uint32_t vm_flags_t;
+typedef uint32_t task_flags_t;
+typedef int64_t ssize_t;
+typedef uint32_t kern_futex_t;
+typedef uint32_t kern_msg_type_t;
+typedef uint32_t kern_msg_event_type_t;
+
+typedef unsigned short equeue_packet_type_t;
+
+typedef unsigned int umode_t;
+
+typedef struct {
+	koid_t obj_id;
+} kern_object_info_t;
+
+typedef struct {
+	virt_addr_t io_base;
+	size_t io_len;
+} kern_iovec_t;
+
+typedef struct {
+	kern_handle_t w_handle;
+	uint32_t w_waitfor;
+	uint32_t w_observed;
+} kern_wait_item_t;
+
+typedef struct {
+	unsigned int hnd_mode;
+	kern_handle_t hnd_value;
+	kern_status_t hnd_result;
+} kern_msg_handle_t;
+
+typedef struct {
+	/* transaction id. identifies a particular request/response exchange.
+	 * used when replying to a particular message. */
+	msgid_t msg_id;
+	/* the id of the task that sent a particular message. */
+	tid_t msg_sender;
+	/* the id of the port or channel used to send a particular message. */
+	koid_t msg_endpoint;
+	/* the message type */
+	kern_msg_type_t msg_type;
+
+	union {
+		/* msg_type = KERN_MSG_TYPE_DATA */
+		struct {
+			/* a list of iovecs that point to the buffers that make
+			 * up the main message data. */
+			kern_iovec_t *msg_data;
+			size_t msg_data_count;
+			/* a list of handle entries that contain the kernel
+			 * handles included in a message. */
+			kern_msg_handle_t *msg_handles;
+			size_t msg_handles_count;
+		};
+
+		/* msg_type = KERN_MSG_TYPE_EVENT */
+		struct {
+			kern_msg_event_type_t msg_event;
+		};
+	};
+} kern_msg_t;
+
+typedef struct {
+	uint32_t s_observed;
+} equeue_packet_async_signal_t;
+
+typedef struct {
+	/* the key of the vm-object for which the page request relates, as
+	 * specified when the vm-object was created */
+	equeue_key_t req_vmo;
+	/* page request type. one of VM_REQUEST_* */
+	unsigned short req_type;
+	/* the offset into the vm-object for which pages are being requested */
+	union {
+		/* used for:
+		 * 	VM_REQUEST_READ
+		 * 	VM_REQUEST_DIRTY
+		 */
+		struct {
+			off_t req_offset;
+			/* the length in bytes of the region being requested */
+			size_t req_length;
+		};
+
+		/* used for:
+		 * 	VM_REQUEST_ATTACH
+		 */
+		struct {
+			/* the key of the original/source vmo. */
+			equeue_key_t req_src_vmo;
+			/* a request ID. used to retrieve information about
+			 * the newly-attached object, as the server won't know
+			 * about it yet, and won't have a handle to it. */
+			uint64_t req_id;
+		};
+	};
+} equeue_packet_vm_request_t;
+
+typedef struct {
+	/* the type of packet. one of EQUEUE_PKT_* */
+	equeue_packet_type_t p_type;
+	/* the key of the object that is responsible for the event, as specified
+	 * when the event was first subscribed to */
+	equeue_key_t p_key;
+
+	union {
+		/* p_type = EQUEUE_PKT_VM_REQUEST */
+		equeue_packet_vm_request_t vm_request;
+		/* p_type = EQUEUE_PKT_ASYNC_SIGNAL */
+		equeue_packet_async_signal_t async_signal;
+	};
+} equeue_packet_t;
+
+#endif

libmango/include-user/mango/handle.h

@@ -1,12 +0,0 @@
-#ifndef MANGO_HANDLE_H_
-#define MANGO_HANDLE_H_
-
-#include <mango/status.h>
-#include <mango/types.h>
-
-extern kern_status_t kern_handle_close(kern_handle_t handle);
-extern kern_status_t kern_handle_duplicate(
-	kern_handle_t handle,
-	kern_handle_t *out);
-
-#endif

libmango/include-user/mango/object.h

@@ -1,8 +0,0 @@
-#ifndef MANGO_OBJECT_H_
-#define MANGO_OBJECT_H_
-
-#include <mango/types.h>
-
-extern kern_status_t kern_object_wait(kern_wait_item_t *items, size_t nr_items);
-
-#endif

libmango/include-user/mango/task.h

@@ -1,29 +0,0 @@
-#ifndef MANGO_TASK_H_
-#define MANGO_TASK_H_
-
-#include <mango/status.h>
-#include <mango/types.h>
-
-extern kern_status_t task_exit(int status);
-extern kern_status_t task_self(kern_handle_t *out);
-
-extern kern_status_t task_create(
-	kern_handle_t parent,
-	const char *name,
-	size_t name_len,
-	kern_handle_t *out_task,
-	kern_handle_t *out_address_space);
-extern kern_status_t task_create_thread(
-	kern_handle_t task,
-	virt_addr_t ip,
-	virt_addr_t sp,
-	uintptr_t *args,
-	size_t nr_args,
-	kern_handle_t *out_thread);
-extern kern_status_t task_get_address_space(
-	kern_handle_t task,
-	kern_handle_t *out);
-
-extern kern_status_t thread_start(kern_handle_t thread);
-
-#endif

libmango/include/mango/syscall.h

@@ -1,45 +0,0 @@
-#ifndef MANGO_SYSCALL_H_
-#define MANGO_SYSCALL_H_
-
-#define SYS_KERN_LOG                    0x00u
-#define SYS_KERN_HANDLE_CLOSE           0x01u
-#define SYS_KERN_HANDLE_DUPLICATE       0x02u
-#define SYS_KERN_CONFIG_GET             0x03u
-#define SYS_KERN_CONFIG_SET             0x04u
-#define SYS_KERN_OBJECT_WAIT            0x05u
-#define SYS_KERN_OBJECT_WAIT_ASYNC      0x06u
-#define SYS_TASK_EXIT                   0x07u
-#define SYS_TASK_SELF                   0x08u
-#define SYS_TASK_CREATE                 0x09u
-#define SYS_TASK_CREATE_THREAD          0x0Au
-#define SYS_TASK_GET_ADDRESS_SPACE      0x0Bu
-#define SYS_THREAD_START                0x0Cu
-#define SYS_VM_OBJECT_CREATE            0x0Du
-#define SYS_VM_OBJECT_READ              0x0Eu
-#define SYS_VM_OBJECT_WRITE             0x0Fu
-#define SYS_VM_OBJECT_COPY              0x10u
-#define SYS_ADDRESS_SPACE_READ          0x11u
-#define SYS_ADDRESS_SPACE_WRITE         0x12u
-#define SYS_ADDRESS_SPACE_MAP           0x13u
-#define SYS_ADDRESS_SPACE_UNMAP         0x14u
-#define SYS_ADDRESS_SPACE_RESERVE       0x15u
-#define SYS_ADDRESS_SPACE_RELEASE       0x16u
-#define SYS_MSG_SEND                    0x17u
-#define SYS_MSG_RECV                    0x18u
-#define SYS_MSG_REPLY                   0x19u
-#define SYS_MSG_READ                    0x1Au
-#define SYS_MSG_WRITE                   0x1Bu
-#define SYS_CHANNEL_CREATE              0x1Cu
-#define SYS_PORT_CREATE                 0x1Du
-#define SYS_PORT_CONNECT                0x1Eu
-#define SYS_PORT_DISCONNECT             0x1Fu
-#define SYS_EQUEUE_CREATE               0x20u
-#define SYS_EQUEUE_DEQUEUE              0x21u
-#define SYS_VM_CONTROLLER_CREATE        0x22u
-#define SYS_VM_CONTROLLER_RECV          0x23u
-#define SYS_VM_CONTROLLER_RECV_ASYNC    0x24u
-#define SYS_VM_CONTROLLER_CREATE_OBJECT 0x25u
-#define SYS_VM_CONTROLLER_DETACH_OBJECT 0x26u
-#define SYS_VM_CONTROLLER_SUPPLY_PAGES  0x27u
-
-#endif

libmango/include/mango/types.h

@@ -1,150 +0,0 @@
-#ifndef MANGO_TYPES_H_
-#define MANGO_TYPES_H_
-
-#include <stddef.h>
-#include <stdint.h>
-
-#define VM_PROT_READ            0x01u
-#define VM_PROT_WRITE           0x02u
-#define VM_PROT_EXEC            0x04u
-#define VM_PROT_USER            0x08u
-#define VM_PROT_SVR             0x10u
-#define VM_PROT_NOCACHE         0x10u
-#define VM_PROT_MAP_SPECIFIC    0x40u
-
-#define MAP_ADDRESS_ANY         ((virt_addr_t) - 1)
-#define MAP_ADDRESS_INVALID     ((virt_addr_t)0)
-#define KERN_HANDLE_INVALID     ((kern_handle_t)0xFFFFFFFF)
-
-#define KERN_CFG_INVALID        0x00u
-#define KERN_CFG_PAGE_SIZE      0x01u
-
-/* maximum number of handles that can be sent in a single message */
-#define KERN_MSG_MAX_HANDLES    64
-
-/* the corresponding handle should be ignored */
-#define KERN_MSG_HANDLE_IGNORE  0
-/* the corresponding handle should be moved to the recipient task. the handle
- * will be closed. */
-#define KERN_MSG_HANDLE_MOVE    1
-/* the corresponding handle should be copied to the recipient task. the handle
- * will remain valid for the sending task. */
-#define KERN_MSG_HANDLE_COPY    2
-
-/* maximum number of objects that can be waited on in a single call to
- * kern_object_wait */
-#define KERN_WAIT_MAX_ITEMS     64
-
-/* equeue packet types */
-#define EQUEUE_PKT_PAGE_REQUEST 0x01u
-#define EQUEUE_PKT_ASYNC_SIGNAL 0x02u
-
-/* page request types */
-#define PAGE_REQUEST_READ       0x01u
-#define PAGE_REQUEST_DIRTY      0x02u
-#define PAGE_REQUEST_DETACH     0x03u
-
-#define IOVEC(p, len)                                                          \
-	{                                                                      \
-		.io_base = (virt_addr_t)(p),                                   \
-		.io_len = (len),                                               \
-	}
-#define MSG_HANDLE(mode, value)                                                \
-	{                                                                      \
-		.hnd_mode = (mode),                                            \
-		.hnd_value = (value),                                          \
-	}
-#define MSG(data, data_count, handles, handles_len)                            \
-	{                                                                      \
-		.msg_data = (data),                                            \
-		.msg_data_count = (data_count),                                \
-		.msg_handles = (handles),                                      \
-		.msg_handles_count = (handles_len),                            \
-	}
-
-typedef uintptr_t phys_addr_t;
-typedef uintptr_t virt_addr_t;
-typedef uint64_t msgid_t;
-typedef uint64_t off_t;
-typedef uint64_t koid_t;
-typedef uintptr_t equeue_key_t;
-typedef unsigned int tid_t;
-typedef unsigned int vm_controller_packet_type_t;
-typedef unsigned int kern_status_t;
-typedef uint32_t kern_handle_t;
-typedef uint32_t kern_config_key_t;
-typedef uint32_t vm_prot_t;
-typedef int64_t ssize_t;
-
-typedef unsigned short equeue_packet_type_t;
-
-typedef unsigned int umode_t;
-
-typedef struct {
-	virt_addr_t io_base;
-	size_t io_len;
-} kern_iovec_t;
-
-typedef struct {
-	kern_handle_t w_handle;
-	uint32_t w_waitfor;
-	uint32_t w_observed;
-} kern_wait_item_t;
-
-typedef struct {
-	unsigned int hnd_mode;
-	kern_handle_t hnd_value;
-	kern_status_t hnd_result;
-} kern_msg_handle_t;
-
-typedef struct {
-	/* transaction id. identifies a particular request/response exchange.
-	 * used when replying to a particular message. */
-	msgid_t msg_id;
-	/* the id of the task that sent a particular message. */
-	tid_t msg_sender;
-	/* the id of the port or channel used to send a particular message. */
-	koid_t msg_endpoint;
-	/* a list of iovecs that point to the buffers that make up the main
-	 * message data. */
-	kern_iovec_t *msg_data;
-	size_t msg_data_count;
-	/* a list of handle entries that contain the kernel handles included
-	 * in a message. */
-	kern_msg_handle_t *msg_handles;
-	size_t msg_handles_count;
-} kern_msg_t;
-
-typedef struct {
-	uint32_t s_observed;
-} equeue_packet_async_signal_t;
-
-typedef struct {
-	/* the key of the vm-object for which the page request relates, as
-	 * specified when the vm-object was created */
-	equeue_key_t req_vmo;
-	/* page request type. one of PAGE_REQUEST_* */
-	unsigned short req_type;
-	/* of the offset into the vm-object for which pages are being requested
-	 */
-	off_t req_offset;
-	/* the length in bytes of the region being requested */
-	size_t req_length;
-} equeue_packet_page_request_t;
-
-typedef struct {
-	/* the type of packet. one of EQUEUE_PKT_* */
-	equeue_packet_type_t p_type;
-	/* the key of the object that is responsible for the event, as specified
-	 * when the event was first subscribed to */
-	equeue_key_t p_key;
-
-	union {
-		/* p_type = EQUEUE_PKT_PAGE_REQUEST */
-		equeue_packet_page_request_t page_request;
-		/* p_type = EQUEUE_PKT_ASYNC_SIGNAL */
-		equeue_packet_async_signal_t async_signal;
-	};
-} equeue_packet_t;
-
-#endif

sched/core.c

@@ -123,8 +123,16 @@ void __schedule(enum sched_mode mode)
 
 	enum thread_state prev_state = READ_ONCE(prev->tr_state);
 
-	if ((mode == SCHED_IRQ || prev_state == THREAD_READY)
-	    && prev != rq->rq_idle) {
+	bool reschedule = false;
+	if (prev_state == THREAD_READY || mode == SCHED_IRQ) {
+		reschedule = true;
+	}
+
+	if (prev == rq->rq_idle || prev_state == THREAD_STOPPED) {
+		reschedule = false;
+	}
+
+	if (reschedule) {
 		rq_enqueue(rq, prev);
 	}
 
@@ -207,18 +215,19 @@ void schedule_thread_on_cpu(struct thread *thr)
 
 void start_charge_period(void)
 {
-	struct thread *self = current_thread();
+	struct thread *self = get_current_thread();
 	if (!self) {
 		return;
 	}
 
 	self->tr_charge_period_start = get_cycles();
+	put_current_thread(self);
 }
 
 void end_charge_period(void)
 {
 	preempt_disable();
-	struct thread *self = current_thread();
+	struct thread *self = get_current_thread();
 	if (!self) {
 		return;
 	}
@@ -236,6 +245,7 @@ void end_charge_period(void)
 	}
 
 	self->tr_charge_period_start = 0;
+	put_current_thread(self);
 
 	// printk("%llu cycles charged to %s/%u", charge,
 	// self->tr_parent->t_name, self->tr_parent->t_id);

sched/task.c

@@ -171,8 +171,13 @@ struct task *task_alloc(void)
 	return t;
 }
 
-struct task *task_create(const char *name, size_t name_len)
+struct task *task_create(
+	struct task *parent,
+	task_flags_t task_flags,
+	const char *name,
+	size_t name_len)
 {
+	kern_status_t status = KERN_OK;
 	struct task *task = task_alloc();
 	if (!task) {
 		return NULL;
@@ -191,9 +196,21 @@ struct task *task_create(const char *name, size_t name_len)
 		VM_USER_LIMIT,
 		&task->t_address_space);
 
+	if (task_flags & TASK_F_CLONE_ALL_HANDLES) {
+		status = handle_table_duplicate(
+			parent->t_handles,
+			&task->t_handles);
+	} else {
+		task->t_handles = handle_table_create();
+	}
+
+	if (status != KERN_OK) {
+		object_unref(&task->t_base);
+		return NULL;
+	}
+
 	task->t_address_space->s_pmap = pmap;
 	task->t_state = TASK_RUNNING;
-	task->t_handles = handle_table_create();
 
 	if (name) {
 		name_len = MIN(name_len, sizeof task->t_name - 1);
@@ -222,13 +239,13 @@ kern_status_t task_add_channel(
 {
 	channel->c_id = id;
 
-	if (!task->b_channels.b_root) {
-		task->b_channels.b_root = &channel->c_node;
-		btree_insert_fixup(&task->b_channels, &channel->c_node);
+	if (!task->t_channels.b_root) {
+		task->t_channels.b_root = &channel->c_node;
+		btree_insert_fixup(&task->t_channels, &channel->c_node);
 		return KERN_OK;
 	}
 
-	struct btree_node *cur = task->b_channels.b_root;
+	struct btree_node *cur = task->t_channels.b_root;
 	while (1) {
 		struct channel *cur_node
 			= BTREE_CONTAINER(struct channel, c_node, cur);
@@ -255,7 +272,7 @@ kern_status_t task_add_channel(
 		cur = next;
 	}
 
-	btree_insert_fixup(&task->b_channels, &channel->c_node);
+	btree_insert_fixup(&task->t_channels, &channel->c_node);
 	return KERN_OK;
 }
 
@@ -268,7 +285,7 @@ BTREE_DEFINE_SIMPLE_GET(
 
 struct channel *task_get_channel(struct task *task, unsigned int id)
 {
-	return get_channel_with_id(&task->b_channels, id);
+	return get_channel_with_id(&task->t_channels, id);
 }
 
 struct task *task_from_tid(tid_t id)
@@ -280,6 +297,109 @@ struct task *task_from_tid(tid_t id)
 	return t;
 }
 
+kern_status_t task_config_get(
+	struct task *task,
+	kern_config_key_t key,
+	void *out,
+	size_t max)
+{
+	switch (key) {
+	case TASK_CFG_ID: {
+		if (max != sizeof(tid_t)) {
+			return KERN_INVALID_ARGUMENT;
+		}
+
+		tid_t *value = out;
+		*value = task->t_id;
+		return KERN_OK;
+	}
+
+	default:
+		return KERN_INVALID_ARGUMENT;
+	}
+}
+
+kern_status_t task_config_set(
+	struct task *task,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len)
+{
+	switch (key) {
+	default:
+		return KERN_INVALID_ARGUMENT;
+	}
+}
+
+void task_exit(int status)
+{
+	struct task *self = get_current_task();
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+	struct task *parent = self->t_parent;
+
+	if (parent) {
+		task_unlock_irqrestore(self, flags);
+		task_lock_irqsave(parent, &flags);
+		task_lock(self);
+		queue_delete(&parent->t_children, &self->t_child_entry);
+		task_unlock(parent);
+	}
+
+	struct thread *cur_thread = get_current_thread();
+
+	self->t_state = TASK_STOPPED;
+	cur_thread->tr_state = THREAD_STOPPED;
+
+	struct queue_entry *cur = queue_first(&self->t_threads);
+	while (cur) {
+		struct queue_entry *next = queue_next(cur);
+		struct thread *thread
+			= QUEUE_CONTAINER(struct thread, tr_parent_entry, cur);
+
+		if (thread == cur_thread) {
+			cur = next;
+			continue;
+		}
+
+		thread_lock(thread);
+		thread_kill(thread);
+		queue_delete(&self->t_threads, cur);
+		thread_unlock(thread);
+
+		cur = next;
+	}
+
+	object_unref(&self->t_address_space->s_base);
+	spin_lock_t *handles_lock = &self->t_handles_lock;
+	struct handle_table *handles = self->t_handles;
+	spin_lock(&self->t_handles_lock);
+
+	pmap_switch(get_kernel_pmap());
+	self->t_address_space->s_pmap = PMAP_INVALID;
+	pmap_destroy(self->t_pmap);
+
+	task_unlock(self);
+	handle_table_destroy(handles);
+
+	tracek("thread %s[%u.%u] killed",
+	       self->t_name,
+	       self->t_id,
+	       cur_thread->tr_id);
+	tracek("task %s[%u] killed (%u)",
+	       self->t_name,
+	       self->t_id,
+	       self->t_base.ob_refcount);
+	spin_unlock_irqrestore(handles_lock, flags);
+
+	put_current_thread(cur_thread);
+	put_current_task(self);
+
+	while (1) {
+		schedule(SCHED_NORMAL);
+	}
+}
+
 kern_status_t task_open_handle(
 	struct task *task,
 	struct object *obj,
@@ -287,13 +407,16 @@ kern_status_t task_open_handle(
 	kern_handle_t *out)
 {
 	struct handle *handle_data = NULL;
-	kern_status_t status
-		= handle_table_alloc_handle(task->t_handles, &handle_data, out);
+	kern_status_t status = handle_table_alloc_handle(
+		task->t_handles,
+		KERN_HANDLE_INVALID,
+		&handle_data,
+		out);
 	if (status != KERN_OK) {
 		return status;
 	}
 
-	object_add_handle(obj);
+	object_ref(obj);
 	handle_data->h_object = obj;
 	handle_data->h_flags = flags;
 
@@ -342,8 +465,19 @@ struct thread *task_create_thread(struct task *parent)
 	return thread;
 }
 
-struct task *current_task(void)
+struct task *get_current_task(void)
 {
-	struct thread *thr = current_thread();
-	return thr ? thr->tr_parent : NULL;
+	struct thread *thr = get_current_thread();
+	if (!thr) {
+		return NULL;
+	}
+
+	struct task *out = task_ref(thr->tr_parent);
+	put_current_thread(thr);
+	return out;
+}
+
+void put_current_task(struct task *task)
+{
+	task_unref(task);
 }

sched/thread.c

@@ -2,15 +2,17 @@
 #include <kernel/cpu.h>
 #include <kernel/machine/thread.h>
 #include <kernel/object.h>
+#include <kernel/printk.h>
 #include <kernel/task.h>
 #include <kernel/thread.h>
+#include <magenta/signal.h>
 
-#define THREAD_CAST(p) OBJECT_C_CAST(struct thread, thr_base, &thread_type, p)
+#define THREAD_CAST(p) OBJECT_C_CAST(struct thread, tr_base, &thread_type, p)
 
 static struct object_type thread_type = {
 	.ob_name = "thread",
 	.ob_size = sizeof(struct thread),
-	.ob_header_offset = offsetof(struct thread, thr_base),
+	.ob_header_offset = offsetof(struct thread, tr_base),
 };
 
 kern_status_t thread_object_type_init(void)
@@ -63,9 +65,6 @@ kern_status_t thread_init_user(
 	const uintptr_t *args,
 	size_t nr_args)
 {
-	thr->tr_id = thr->tr_parent->t_next_thread_id++;
-
-	thr->tr_prio = PRIO_NORMAL;
 	thr->tr_state = THREAD_READY;
 	thr->tr_quantum_target = default_quantum();
 
@@ -102,11 +101,57 @@ kern_status_t thread_init_user(
 	return KERN_OK;
 }
 
+kern_status_t thread_init_user_clone(
+	struct thread *thr,
+	const struct thread *src,
+	uintptr_t return_value)
+{
+	thr->tr_state = THREAD_READY;
+	thr->tr_quantum_target = default_quantum();
+
+	thr->tr_kstack = vm_page_alloc(THREAD_KSTACK_ORDER, VM_NORMAL);
+	if (!thr->tr_kstack) {
+		return KERN_NO_MEMORY;
+	}
+
+	thr->tr_sp = (uintptr_t)vm_page_get_vaddr(thr->tr_kstack)
+	           + vm_page_order_to_bytes(THREAD_KSTACK_ORDER);
+	thr->tr_bp = thr->tr_sp;
+	thr->tr_cpu_kernel_sp = thr->tr_sp;
+
+	/* the new thread needs two contextx:
+	 *   1) to get the thread running in kernel mode, so that it can
+	 *   execute ml_thread_switch_user
+	 *   2) to allow ml_thread_switch_user to jump to the correct place
+	 *   in usermode (and with the correct stack).
+	 *
+	 * these two contexts are constructed on the thread's kernel stack
+	 * in reverse order.
+	 */
+
+	/* this context will be used by ml_user_return to jump to userspace
+	 * with the specified instruction pointer and user stack */
+	ml_thread_clone_user_context(
+		src->tr_irqctx,
+		&src->tr_ml,
+		&thr->tr_ml,
+		return_value,
+		&thr->tr_sp);
+	/* this context will be used by the scheduler and ml_thread_switch to
+	 * jump to ml_user_return in kernel mode with the thread's kernel stack.
+	 */
+	ml_thread_prepare_kernel_context(
+		(uintptr_t)ml_thread_switch_user,
+		&thr->tr_sp);
+
+	return KERN_OK;
+}
+
 void thread_free(struct thread *thr)
 {
 }
 
-struct thread *current_thread(void)
+struct thread *get_current_thread(void)
 {
 	struct cpu_data *cpu = get_this_cpu();
 	if (!cpu) {
@@ -114,13 +159,24 @@ struct thread *current_thread(void)
 	}
 
 	struct thread *out = cpu->c_rq.rq_cur;
+	object_ref(&out->tr_base);
+
 	put_cpu(cpu);
 	return out;
 }
 
+void put_current_thread(struct thread *thr)
+{
+	object_unref(&thr->tr_base);
+}
+
 bool need_resched(void)
 {
-	return (current_thread()->tr_flags & THREAD_F_NEED_RESCHED) != 0;
+	struct thread *thr = get_current_thread();
+	bool result = (thr->tr_flags & THREAD_F_NEED_RESCHED) != 0;
+	put_current_thread(thr);
+
+	return result;
 }
 
 int thread_priority(struct thread *thr)
@@ -142,6 +198,56 @@ void thread_awaken(struct thread *thr)
 	rq_unlock(rq, flags);
 }
 
+void thread_exit(void)
+{
+	struct thread *self = get_current_thread();
+	unsigned long flags;
+	thread_lock_irqsave(self, &flags);
+	self->tr_state = THREAD_STOPPED;
+	object_assert_signal(&self->tr_base, THREAD_SIGNAL_STOPPED);
+	tracek("thread %s[%u.%u] exited",
+	       self->tr_parent->t_name,
+	       self->tr_parent->t_id,
+	       self->tr_id);
+	thread_unlock_irqrestore(self, flags);
+	put_current_thread(self);
+
+	while (1) {
+		schedule(SCHED_NORMAL);
+	}
+}
+
+void thread_join(struct thread *thread, unsigned long *irq_flags)
+{
+	while (1) {
+		if (thread->tr_state == THREAD_STOPPED) {
+			break;
+		}
+
+		object_wait_signal(
+			&thread->tr_base,
+			THREAD_SIGNAL_STOPPED,
+			irq_flags);
+	}
+}
+
+void thread_kill(struct thread *thread)
+{
+	thread->tr_state = THREAD_STOPPED;
+	if (thread->tr_rq) {
+		unsigned long flags;
+		rq_lock(thread->tr_rq, &flags);
+		rq_remove_thread(thread->tr_rq, thread);
+		rq_unlock(thread->tr_rq, flags);
+	}
+
+	object_assert_signal(&thread->tr_base, THREAD_SIGNAL_STOPPED);
+	tracek("thread %s[%u.%u] killed",
+	       thread->tr_parent->t_name,
+	       thread->tr_parent->t_id,
+	       thread->tr_id);
+}
+
 struct thread *create_kernel_thread(void (*fn)(void))
 {
 	struct task *kernel = kernel_task();
@@ -185,3 +291,31 @@ struct thread *create_idle_thread(void)
 
 	return thr;
 }
+
+kern_status_t thread_config_get(
+	struct thread *thread,
+	kern_config_key_t key,
+	void *out,
+	size_t max)
+{
+	switch (key) {
+	default:
+		break;
+	}
+
+	return ml_thread_config_get(thread, key, out, max);
+}
+
+kern_status_t thread_config_set(
+	struct thread *thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len)
+{
+	switch (key) {
+	default:
+		break;
+	}
+
+	return ml_thread_config_set(thread, key, ptr, len);
+}

sched/timer.c

@@ -44,7 +44,7 @@ unsigned long schedule_timeout(unsigned long ticks)
 {
 	struct timer timer;
 
-	struct thread *self = current_thread();
+	struct thread *self = get_current_thread();
 
 	timer.t_entry = QUEUE_ENTRY_INIT;
 	timer.t_expiry = clock_ticks + ticks;
@@ -58,6 +58,7 @@ unsigned long schedule_timeout(unsigned long ticks)
 	schedule(SCHED_NORMAL);
 
 	remove_timer(&timer);
+	put_current_thread(self);
 
 	return 0;
 }

sched/wait.c

@@ -69,6 +69,24 @@ void wakeup_queue(struct waitqueue *q)
 	spin_unlock_irqrestore(&q->wq_lock, flags);
 }
 
+void wakeup_n(struct waitqueue *q, size_t nr_waiters)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&q->wq_lock, &flags);
+	struct queue_entry *ent = queue_pop_front(&q->wq_waiters);
+	while (ent && nr_waiters > 0) {
+		struct wait_item *waiter
+			= QUEUE_CONTAINER(struct wait_item, w_entry, ent);
+		struct thread *thr = waiter->w_thread;
+		thread_awaken(thr);
+
+		ent = queue_pop_front(&q->wq_waiters);
+		nr_waiters--;
+	}
+
+	spin_unlock_irqrestore(&q->wq_lock, flags);
+}
+
 void wakeup_one(struct waitqueue *q)
 {
 	unsigned long flags;
@@ -92,7 +110,7 @@ void wakeup_one(struct waitqueue *q)
 
 void sleep_forever(void)
 {
-	struct thread *thr = current_thread();
+	struct thread *thr = get_current_thread();
 	struct runqueue *rq = thr->tr_rq;
 
 	unsigned long flags;
@@ -103,7 +121,17 @@ void sleep_forever(void)
 
 	rq_unlock(rq, flags);
 
-	while (thr->tr_state == THREAD_SLEEPING) {
+	put_current_thread(thr);
+
+	while (1) {
+		thr = get_current_thread();
+		bool sleep = (thr->tr_state == THREAD_SLEEPING);
+		put_current_thread(thr);
+
+		if (!sleep) {
+			break;
+		}
+
 		schedule(SCHED_NORMAL);
 	}
 }

syscall/address-space.c

@@ -11,13 +11,15 @@ kern_status_t sys_address_space_read(
 	size_t count,
 	size_t *nr_read)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_access_w(self, dst, count)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (nr_read && !validate_access_w(self, nr_read, sizeof *nr_read)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -30,12 +32,14 @@ kern_status_t sys_address_space_read(
 		= task_resolve_handle(self, region_handle, &obj, &handle_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
 	struct address_space *region = address_space_cast(obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -52,6 +56,7 @@ kern_status_t sys_address_space_read(
 	address_space_unlock_irqrestore(region, flags);
 
 	object_unref(obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -63,14 +68,16 @@ kern_status_t sys_address_space_write(
 	size_t count,
 	size_t *nr_written)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_access_r(self, src, count)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (nr_written
 	    && !validate_access_w(self, nr_written, sizeof *nr_written)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -83,12 +90,14 @@ kern_status_t sys_address_space_write(
 		= task_resolve_handle(self, region_handle, &obj, &handle_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
 	struct address_space *region = address_space_cast(obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -105,6 +114,7 @@ kern_status_t sys_address_space_write(
 	address_space_unlock_irqrestore(region, flags);
 
 	object_unref(obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -115,22 +125,24 @@ kern_status_t sys_address_space_map(
 	kern_handle_t object_handle,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out_base_address)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (out_base_address
 	    && !validate_access_r(
 		    self,
 		    out_base_address,
 		    sizeof *out_base_address)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	kern_status_t status = KERN_OK;
-	unsigned long flags;
-	task_lock_irqsave(self, &flags);
+	unsigned long irq_flags;
+	task_lock_irqsave(self, &irq_flags);
 
 	struct object *region_obj = NULL, *vmo_obj = NULL;
 	handle_flags_t region_flags = 0, vmo_flags = 0;
@@ -140,30 +152,34 @@ kern_status_t sys_address_space_map(
 		&region_obj,
 		&region_flags);
 	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
+		put_current_task(self);
 		return status;
 	}
 
 	status = task_resolve_handle(self, object_handle, &vmo_obj, &vmo_flags);
 	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
+		put_current_task(self);
 		return status;
 	}
 
 	struct address_space *region = address_space_cast(region_obj);
 	if (!region) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
 	struct vm_object *vmo = vm_object_cast(vmo_obj);
 	if (!vmo) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	task_unlock_irqrestore(self, flags);
-	address_space_lock_irqsave(region, &flags);
+	task_unlock_irqrestore(self, irq_flags);
+	address_space_lock_irqsave(region, &irq_flags);
 	/* address_space_map will take care of locking `vmo` */
 	status = address_space_map(
 		region,
@@ -171,12 +187,14 @@ kern_status_t sys_address_space_map(
 		vmo,
 		object_offset,
 		length,
+		flags,
 		prot,
 		out_base_address);
-	address_space_unlock_irqrestore(region, flags);
+	address_space_unlock_irqrestore(region, irq_flags);
 
 	object_unref(vmo_obj);
 	object_unref(region_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -186,7 +204,7 @@ kern_status_t sys_address_space_unmap(
 	virt_addr_t base,
 	size_t length)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	kern_status_t status = KERN_OK;
 	unsigned long flags;
@@ -201,12 +219,14 @@ kern_status_t sys_address_space_unmap(
 		&region_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
 	struct address_space *region = address_space_cast(region_obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -215,6 +235,7 @@ kern_status_t sys_address_space_unmap(
 	status = address_space_unmap(region, base, length);
 
 	object_unref(region_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -225,13 +246,14 @@ kern_status_t sys_address_space_reserve(
 	size_t length,
 	virt_addr_t *out_base_address)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (out_base_address
 	    && !validate_access_r(
 		    self,
 		    out_base_address,
 		    sizeof *out_base_address)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -248,12 +270,14 @@ kern_status_t sys_address_space_reserve(
 		&region_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
 	struct address_space *region = address_space_cast(region_obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -268,6 +292,7 @@ kern_status_t sys_address_space_reserve(
 	address_space_unlock_irqrestore(region, flags);
 
 	object_unref(region_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -277,7 +302,7 @@ kern_status_t sys_address_space_release(
 	virt_addr_t base,
 	size_t length)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	kern_status_t status = KERN_OK;
 	unsigned long flags;
@@ -292,12 +317,14 @@ kern_status_t sys_address_space_release(
 		&region_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
 	struct address_space *region = address_space_cast(region_obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -308,6 +335,7 @@ kern_status_t sys_address_space_release(
 	address_space_unlock_irqrestore(region, flags);
 
 	object_unref(region_obj);
+	put_current_task(self);
 
 	return status;
 }

syscall/config.c

@@ -3,21 +3,26 @@
 
 kern_status_t sys_kern_config_get(kern_config_key_t key, void *ptr, size_t len)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
+	kern_status_t status = KERN_OK;
 
 	switch (key) {
 	case KERN_CFG_PAGE_SIZE:
 		if (!validate_access_w(self, ptr, sizeof(uintptr_t))) {
-			return KERN_MEMORY_FAULT;
+			status = KERN_MEMORY_FAULT;
+			break;
 		}
 
 		*(uint32_t *)ptr = VM_PAGE_SIZE;
-		return KERN_OK;
+		status = KERN_OK;
+		break;
 	default:
-		return KERN_INVALID_ARGUMENT;
+		status = KERN_INVALID_ARGUMENT;
+		break;
 	}
 
-	return KERN_UNSUPPORTED;
+	put_current_task(self);
+	return status;
 }
 
 kern_status_t sys_kern_config_set(

syscall/dispatch.c

@@ -10,7 +10,14 @@ static const virt_addr_t syscall_table[] = {
 	SYSCALL_TABLE_ENTRY(TASK_CREATE, task_create),
 	SYSCALL_TABLE_ENTRY(TASK_CREATE_THREAD, task_create_thread),
 	SYSCALL_TABLE_ENTRY(TASK_GET_ADDRESS_SPACE, task_get_address_space),
+	SYSCALL_TABLE_ENTRY(TASK_DUPLICATE, task_duplicate),
+	SYSCALL_TABLE_ENTRY(TASK_CONFIG_GET, task_config_get),
+	SYSCALL_TABLE_ENTRY(TASK_CONFIG_SET, task_config_set),
+	SYSCALL_TABLE_ENTRY(THREAD_SELF, thread_self),
 	SYSCALL_TABLE_ENTRY(THREAD_START, thread_start),
+	SYSCALL_TABLE_ENTRY(THREAD_EXIT, thread_exit),
+	SYSCALL_TABLE_ENTRY(THREAD_CONFIG_GET, thread_config_get),
+	SYSCALL_TABLE_ENTRY(THREAD_CONFIG_SET, thread_config_set),
 	SYSCALL_TABLE_ENTRY(VM_OBJECT_CREATE, vm_object_create),
 	SYSCALL_TABLE_ENTRY(VM_OBJECT_READ, vm_object_read),
 	SYSCALL_TABLE_ENTRY(VM_OBJECT_WRITE, vm_object_write),
@@ -23,7 +30,8 @@ static const virt_addr_t syscall_table[] = {
 	SYSCALL_TABLE_ENTRY(ADDRESS_SPACE_RELEASE, address_space_release),
 	SYSCALL_TABLE_ENTRY(KERN_LOG, kern_log),
 	SYSCALL_TABLE_ENTRY(KERN_HANDLE_CLOSE, kern_handle_close),
-	SYSCALL_TABLE_ENTRY(KERN_HANDLE_DUPLICATE, kern_handle_duplicate),
+	SYSCALL_TABLE_ENTRY(KERN_HANDLE_TRANSFER, kern_handle_transfer),
+	SYSCALL_TABLE_ENTRY(KERN_HANDLE_CONTROL, kern_handle_control),
 	SYSCALL_TABLE_ENTRY(KERN_CONFIG_GET, kern_config_get),
 	SYSCALL_TABLE_ENTRY(KERN_CONFIG_SET, kern_config_set),
 	SYSCALL_TABLE_ENTRY(CHANNEL_CREATE, channel_create),
@@ -41,6 +49,12 @@ static const virt_addr_t syscall_table[] = {
 	SYSCALL_TABLE_ENTRY(
 		VM_CONTROLLER_CREATE_OBJECT,
 		vm_controller_create_object),
+	SYSCALL_TABLE_ENTRY(
+		VM_CONTROLLER_PREPARE_ATTACH,
+		vm_controller_prepare_attach),
+	SYSCALL_TABLE_ENTRY(
+		VM_CONTROLLER_FINISH_ATTACH,
+		vm_controller_finish_attach),
 	SYSCALL_TABLE_ENTRY(
 		VM_CONTROLLER_DETACH_OBJECT,
 		vm_controller_detach_object),
@@ -48,6 +62,9 @@ static const virt_addr_t syscall_table[] = {
 		VM_CONTROLLER_SUPPLY_PAGES,
 		vm_controller_supply_pages),
 	SYSCALL_TABLE_ENTRY(KERN_OBJECT_WAIT, kern_object_wait),
+	SYSCALL_TABLE_ENTRY(KERN_OBJECT_QUERY, kern_object_query),
+	SYSCALL_TABLE_ENTRY(FUTEX_WAIT, futex_wait),
+	SYSCALL_TABLE_ENTRY(FUTEX_WAKE, futex_wake),
 };
 static const size_t syscall_table_count
 	= sizeof syscall_table / sizeof syscall_table[0];

syscall/futex.c

@@ -0,0 +1,41 @@
+#include <kernel/futex.h>
+#include <kernel/sched.h>
+#include <kernel/syscall.h>
+#include <kernel/task.h>
+
+kern_status_t sys_futex_wait(
+	kern_futex_t *futex,
+	kern_futex_t new_val,
+	unsigned int flags)
+{
+	struct task *self = get_current_task();
+	if (!validate_access_r(self, futex, sizeof *futex)) {
+		put_current_task(self);
+		return KERN_MEMORY_FAULT;
+	}
+
+	futex_key_t key;
+	kern_status_t status = futex_get(futex, &key, flags);
+	if (status != KERN_OK) {
+		put_current_task(self);
+		return status;
+	}
+
+	status = futex_wait(key, new_val, flags);
+	put_current_task(self);
+	return status;
+}
+
+kern_status_t sys_futex_wake(
+	kern_futex_t *futex,
+	unsigned int nr_waiters,
+	unsigned int flags)
+{
+	futex_key_t key;
+	kern_status_t status = futex_get(futex, &key, flags);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	return futex_wake(key, nr_waiters, flags);
+}

syscall/handle.c

@@ -3,37 +3,158 @@
 
 kern_status_t sys_kern_handle_close(kern_handle_t handle)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
-	return task_close_handle(self, handle);
+	kern_status_t status = task_close_handle(self, handle);
+
+	put_current_task(self);
+
+	return status;
 }
 
-kern_status_t sys_kern_handle_duplicate(
-	kern_handle_t handle,
-	kern_handle_t *out)
+kern_status_t sys_kern_handle_transfer(
+	kern_handle_t src_task_handle,
+	kern_handle_t src_handle,
+	kern_handle_t dest_task_handle,
+	kern_handle_t dest_handle,
+	unsigned int mode,
+	kern_handle_t *out_handle)
 {
-	struct task *self = current_task();
+	switch (mode) {
+	case HANDLE_TRANSFER_MOVE:
+	case HANDLE_TRANSFER_COPY:
+		break;
+	default:
+		return KERN_INVALID_ARGUMENT;
+	}
 
-	if (!validate_access_w(self, out, sizeof *out)) {
+	struct task *self = get_current_task();
+
+	if (out_handle
+	    && !validate_access_w(self, out_handle, sizeof *out_handle)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	unsigned long flags;
 	task_lock_irqsave(self, &flags);
 
+	struct object *src_object = NULL;
+	struct task *src_task = NULL;
+	struct task *dest_task = NULL;
+	kern_status_t status = KERN_OK;
+
 	struct object *obj = NULL;
 	handle_flags_t handle_flags = 0;
-	kern_status_t status
-		= task_resolve_handle(self, handle, &obj, &handle_flags);
+
+	if (src_task_handle != KERN_HANDLE_INVALID) {
+		status = task_resolve_handle(
+			self,
+			src_task_handle,
+			&obj,
+			&handle_flags);
 		if (status != KERN_OK) {
 			task_unlock_irqrestore(self, flags);
-		return status;
+			goto cleanup;
 		}
 
-	status = task_open_handle(self, obj, handle_flags, out);
-	object_unref(obj);
+		src_task = task_cast(obj);
+		if (!src_task) {
+			status = KERN_INVALID_ARGUMENT;
+			task_unlock_irqrestore(self, flags);
+			goto cleanup;
+		}
+	} else {
+		src_task = self;
+	}
 
+	if (dest_task_handle != KERN_HANDLE_INVALID) {
+		status = task_resolve_handle(
+			self,
+			dest_task_handle,
+			&obj,
+			&handle_flags);
+		if (status != KERN_OK) {
+			task_unlock_irqrestore(self, flags);
+			goto cleanup;
+		}
+
+		dest_task = task_cast(obj);
+		if (!dest_task) {
+			status = KERN_INVALID_ARGUMENT;
+			task_unlock_irqrestore(self, flags);
+			goto cleanup;
+		}
+	} else {
+		dest_task = self;
+	}
+
+	status = task_resolve_handle(
+		self,
+		src_handle,
+		&src_object,
+		&handle_flags);
 	task_unlock_irqrestore(self, flags);
 
+	if (status != KERN_OK) {
+		goto cleanup;
+	}
+
+	struct handle *dest = NULL;
+	task_lock_irqsave(dest_task, &flags);
+	status = handle_table_alloc_handle(
+		dest_task->t_handles,
+		dest_handle,
+		&dest,
+		&dest_handle);
+	task_unlock_irqrestore(dest_task, flags);
+
+	if (status != KERN_OK) {
+		goto cleanup;
+	}
+
+	if (mode == HANDLE_TRANSFER_MOVE) {
+		object_ref(src_object);
+		task_lock_irqsave(src_task, &flags);
+		handle_table_free_handle(src_task->t_handles, src_handle);
+		task_unlock_irqrestore(src_task, flags);
+	}
+
+	dest->h_object = src_object;
+	dest->h_flags = handle_flags;
+
+	if (out_handle) {
+		*out_handle = dest_handle;
+	}
+
+	put_current_task(self);
+
+	return KERN_OK;
+
+cleanup:
+	if (src_task && src_task_handle != KERN_HANDLE_INVALID) {
+		object_unref(&src_task->t_base);
+	}
+
+	if (dest_task && dest_task_handle != KERN_HANDLE_INVALID) {
+		object_unref(&dest_task->t_base);
+	}
+
+	if (src_object) {
+		object_unref(src_object);
+	}
+
+	put_current_task(self);
+
 	return status;
 }
+
+kern_status_t sys_kern_handle_control(
+	kern_handle_t task,
+	kern_handle_t handle,
+	uint32_t set_mask,
+	uint32_t clear_mask,
+	uint32_t *out_flags)
+{
+	return KERN_UNIMPLEMENTED;
+}

syscall/log.c

@@ -1,10 +1,18 @@
 #include <kernel/printk.h>
 #include <kernel/sched.h>
 #include <kernel/task.h>
+#include <kernel/thread.h>
 
 kern_status_t sys_kern_log(const char *s)
 {
-	struct task *task = current_task();
-	printk("%s[%d]: %s", task->t_name, task->t_id, s);
+#ifdef TRACE
+	struct task *task = get_current_task();
+	struct thread *thread = get_current_thread();
+	printk("%s[%d.%d]: %s", task->t_name, task->t_id, thread->tr_id, s);
+	put_current_thread(thread);
+	put_current_task(task);
+#else
+	printk("%s", s);
+#endif
 	return KERN_OK;
 }

syscall/msg.c

@@ -7,13 +7,15 @@
 
 kern_status_t sys_channel_create(unsigned int id, kern_handle_t *out)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 	if (!validate_access_w(self, out, sizeof *out)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	struct channel *channel = channel_create();
 	if (!channel) {
+		put_current_task(self);
 		return KERN_NO_MEMORY;
 	}
 
@@ -22,6 +24,7 @@ kern_status_t sys_channel_create(unsigned int id, kern_handle_t *out)
 
 	if (task_get_channel(self, id)) {
 		task_unlock_irqrestore(self, irq_flags);
+		put_current_task(self);
 		return KERN_NAME_EXISTS;
 	}
 
@@ -31,11 +34,13 @@ kern_status_t sys_channel_create(unsigned int id, kern_handle_t *out)
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, irq_flags);
 		object_unref(&channel->c_base);
+		put_current_task(self);
 		return status;
 	}
 
 	task_add_channel(self, channel, id);
 	task_unlock_irqrestore(self, irq_flags);
+	put_current_task(self);
 
 	*out = handle;
 	return KERN_OK;
@@ -43,13 +48,15 @@ kern_status_t sys_channel_create(unsigned int id, kern_handle_t *out)
 
 kern_status_t sys_port_create(kern_handle_t *out)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 	if (!validate_access_w(self, out, sizeof *out)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	struct port *port = port_create();
 	if (!port) {
+		put_current_task(self);
 		return KERN_NO_MEMORY;
 	}
 
@@ -60,9 +67,10 @@ kern_status_t sys_port_create(kern_handle_t *out)
 	kern_status_t status
 		= task_open_handle(self, &port->p_base, 0, &handle);
 	task_unlock_irqrestore(self, irq_flags);
+	object_unref(&port->p_base);
+	put_current_task(self);
 
 	if (status != KERN_OK) {
-		object_unref(&port->p_base);
 		return status;
 	}
 
@@ -77,7 +85,7 @@ kern_status_t sys_port_connect(
 {
 	unsigned long flags;
 
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 	task_lock_irqsave(self, &flags);
 
 	struct object *port_obj = NULL;
@@ -88,6 +96,7 @@ kern_status_t sys_port_connect(
 		&port_obj,
 		&port_handle_flags);
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
@@ -96,6 +105,7 @@ kern_status_t sys_port_connect(
 
 	struct task *remote_task = task_from_tid(task_id);
 	if (!remote_task) {
+		put_current_task(self);
 		return KERN_NO_ENTRY;
 	}
 
@@ -104,6 +114,7 @@ kern_status_t sys_port_connect(
 	struct channel *remote = task_get_channel(remote_task, channel_id);
 	if (!remote) {
 		task_unlock_irqrestore(remote_task, flags);
+		put_current_task(self);
 		return KERN_NO_ENTRY;
 	}
 
@@ -114,6 +125,8 @@ kern_status_t sys_port_connect(
 	status = port_connect(port, remote);
 	port_unlock_irqrestore(port, flags);
 	object_unref(&remote->c_base);
+	object_unref(port_obj);
+	put_current_task(self);
 
 	return KERN_OK;
 }
@@ -122,7 +135,7 @@ kern_status_t sys_port_disconnect(kern_handle_t port_handle)
 {
 	unsigned long flags;
 
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 	task_lock_irqsave(self, &flags);
 
 	struct object *port_obj = NULL;
@@ -133,6 +146,7 @@ kern_status_t sys_port_disconnect(kern_handle_t port_handle)
 		&port_obj,
 		&port_handle_flags);
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
@@ -141,6 +155,7 @@ kern_status_t sys_port_disconnect(kern_handle_t port_handle)
 	struct port *port = port_cast(port_obj);
 	if (!port) {
 		object_unref(port_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -148,6 +163,7 @@ kern_status_t sys_port_disconnect(kern_handle_t port_handle)
 	port_lock_irqsave(port, &flags);
 	status = port_disconnect(port);
 	port_unlock_irqrestore(port, flags);
+	put_current_task(self);
 
 	return status;
 }
@@ -218,13 +234,15 @@ kern_status_t sys_msg_send(
 	const kern_msg_t *msg,
 	kern_msg_t *out_reply)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_msg(self, msg, false)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (!validate_msg(self, out_reply, true)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -239,15 +257,17 @@ kern_status_t sys_msg_send(
 		port_handle,
 		&port_obj,
 		&port_handle_flags);
+	task_unlock_irqrestore(self, flags);
+
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
-	task_unlock_irqrestore(self, flags);
-
 	struct port *port = port_cast(port_obj);
 	if (!port) {
 		object_unref(port_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -255,15 +275,17 @@ kern_status_t sys_msg_send(
 	status = port_send_msg(port, msg, out_reply, &flags);
 	port_unlock_irqrestore(port, flags);
 	object_unref(port_obj);
+	put_current_task(self);
 
 	return status;
 }
 
 kern_status_t sys_msg_recv(kern_handle_t channel_handle, kern_msg_t *out_msg)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_msg(self, out_msg, true)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -279,6 +301,7 @@ kern_status_t sys_msg_recv(kern_handle_t channel_handle, kern_msg_t *out_msg)
 		&channel_obj,
 		&channel_handle_flags);
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
@@ -287,6 +310,7 @@ kern_status_t sys_msg_recv(kern_handle_t channel_handle, kern_msg_t *out_msg)
 	struct channel *channel = channel_cast(channel_obj);
 	if (!channel) {
 		object_unref(channel_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -294,6 +318,7 @@ kern_status_t sys_msg_recv(kern_handle_t channel_handle, kern_msg_t *out_msg)
 	status = channel_recv_msg(channel, out_msg, &flags);
 	channel_unlock_irqrestore(channel, flags);
 	object_unref(channel_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -303,9 +328,10 @@ kern_status_t sys_msg_reply(
 	msgid_t id,
 	const kern_msg_t *reply)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_msg(self, reply, true)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -321,6 +347,7 @@ kern_status_t sys_msg_reply(
 		&channel_obj,
 		&channel_handle_flags);
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
@@ -329,6 +356,7 @@ kern_status_t sys_msg_reply(
 	struct channel *channel = channel_cast(channel_obj);
 	if (!channel) {
 		object_unref(channel_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -336,6 +364,7 @@ kern_status_t sys_msg_reply(
 	status = channel_reply_msg(channel, id, reply, &flags);
 	channel_unlock_irqrestore(channel, flags);
 	object_unref(channel_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -348,13 +377,15 @@ kern_status_t sys_msg_read(
 	size_t iov_count,
 	size_t *nr_read)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (nr_read && !validate_access_w(self, nr_read, sizeof *nr_read)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (!validate_iovec(self, iov, iov_count, true)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -370,6 +401,7 @@ kern_status_t sys_msg_read(
 		&channel_obj,
 		&channel_handle_flags);
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
@@ -378,6 +410,7 @@ kern_status_t sys_msg_read(
 	struct channel *channel = channel_cast(channel_obj);
 	if (!channel) {
 		object_unref(channel_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -392,6 +425,7 @@ kern_status_t sys_msg_read(
 		nr_read);
 	channel_unlock_irqrestore(channel, flags);
 	object_unref(channel_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -404,14 +438,16 @@ kern_status_t sys_msg_write(
 	size_t iov_count,
 	size_t *nr_written)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (nr_written
 	    && !validate_access_w(self, nr_written, sizeof *nr_written)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (!validate_iovec(self, iov, iov_count, false)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -427,6 +463,7 @@ kern_status_t sys_msg_write(
 		&channel_obj,
 		&channel_handle_flags);
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
@@ -435,6 +472,7 @@ kern_status_t sys_msg_write(
 	struct channel *channel = channel_cast(channel_obj);
 	if (!channel) {
 		object_unref(channel_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -449,6 +487,7 @@ kern_status_t sys_msg_write(
 		nr_written);
 	channel_unlock_irqrestore(channel, flags);
 	object_unref(channel_obj);
+	put_current_task(self);
 
 	return status;
 }